diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index e38e56127c5..b7373d7b145 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -330,6 +330,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add module for ingesting Cisco FTD logs over syslog. {pull}13286[13286] - Update CoreDNS module to populate ECS DNS fields. {issue}13320[13320] {pull}13505[13505] - Parse query steps in PostgreSQL slowlogs. {issue}13496[13496] {pull}13701[13701] +- Add filebeat azure module with activitylogs, auditlogs, signinlogs filesets. {pull}13776[13776] - Add support to set the document id in the json reader. {pull}5844[5844] - Add input httpjson. {issue}13545[13545] {pull}13546[13546] - Filebeat Netflow input: Remove beta label. {pull}13858[13858] diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index e92437a3a08..203003bdb35 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -15,6 +15,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -1237,6 +1238,921 @@ type: keyword The Transport Layer Security (TLS) version negotiated by the client. +type: keyword + +-- + +[[exported-fields-azure]] +== Azure fields + +Azure Module + + + +[float] +=== azure + + + + +*`azure.resource_id`*:: ++ +-- +Resource ID + + +type: keyword + +-- + +[float] +=== activitylogs + +Fields for Azure activity logs. + + + +[float] +=== identity + +The canonical user ID of the owner of the source bucket. + + + +[float] +=== authorization + +Node allocatable pods + + + +[float] +=== evidence + +Node allocatable pods + + + +*`azure.activitylogs.identity.authorization.evidence.role_assignment_scope`*:: ++ +-- +Role assignment scope + + +type: keyword + +-- + +*`azure.activitylogs.identity.authorization.evidence.role_definition_id`*:: ++ +-- +Role definition ID + + +type: keyword + +-- + +*`azure.activitylogs.identity.authorization.evidence.role`*:: ++ +-- +Role + + +type: keyword + +-- + +*`azure.activitylogs.identity.authorization.evidence.role_assignment_id`*:: ++ +-- +Role assignment ID + + +type: keyword + +-- + +*`azure.activitylogs.identity.authorization.evidence.principal_id`*:: ++ +-- +Principal ID + + +type: keyword + +-- + +*`azure.activitylogs.identity.authorization.evidence.principal_type`*:: ++ +-- +Principal type + + +type: keyword + +-- + +*`azure.activitylogs.identity.scope`*:: ++ +-- +Scope + + +type: keyword + +-- + +*`azure.activitylogs.identity.action`*:: ++ +-- +Action + + +type: keyword + +-- + +*`azure.activitylogs.correlation_id`*:: ++ +-- +Correlation ID + + +type: keyword + +-- + +*`azure.activitylogs.operation_name`*:: ++ +-- +Operation name + + +type: keyword + +-- + +*`azure.activitylogs.result_signature`*:: ++ +-- +Result signature + + +type: keyword + +-- + +[float] +=== properties + +Properties + + + +*`azure.activitylogs.properties.service_request_id`*:: ++ +-- +Service Request Id + + +type: keyword + +-- + +*`azure.activitylogs.properties.status_code`*:: ++ +-- +Status code + + +type: keyword + +-- + +[float] +=== auditlogs + +Fields for Azure audit logs. + + + +*`azure.auditlogs.operation_name`*:: ++ +-- +The operation name + + +type: keyword + +-- + +*`azure.auditlogs.operation_version`*:: ++ +-- +The operation version + + +type: keyword + +-- + +*`azure.auditlogs.tenant_id`*:: ++ +-- +Tenant ID + + +type: keyword + +-- + +*`azure.auditlogs.result_signature`*:: ++ +-- +Result signature + + +type: keyword + +-- + +*`azure.auditlogs.correlation_id`*:: ++ +-- +Correlation ID + + +type: keyword + +-- + +[float] +=== properties + +The audit log properties + + + +*`azure.auditlogs.properties.result`*:: ++ +-- +Log result + + +type: keyword + +-- + +*`azure.auditlogs.properties.activity_display_name`*:: ++ +-- +Activity display name + + +type: keyword + +-- + +*`azure.auditlogs.properties.result_reason`*:: ++ +-- +Reason for the log result + + +type: keyword + +-- + +*`azure.auditlogs.properties.correlation_id`*:: ++ +-- +Correlation ID + + +type: keyword + +-- + +*`azure.auditlogs.properties.logged_by_service`*:: ++ +-- +Logged by service + + +type: keyword + +-- + +*`azure.auditlogs.properties.operation_type`*:: ++ +-- +Operation type + + +type: keyword + +-- + +*`azure.auditlogs.properties.id`*:: ++ +-- +ID + + +type: keyword + +-- + +*`azure.auditlogs.properties.activityDateTime`*:: ++ +-- +Activity timestamp + + +type: keyword + +-- + +*`azure.auditlogs.properties.category`*:: ++ +-- +category + + +type: keyword + +-- + +[float] +=== target_resources + +Target resources + + + +*`azure.auditlogs.properties.target_resources.display_name`*:: ++ +-- +Display name + + +type: keyword + +-- + +*`azure.auditlogs.properties.target_resources.id`*:: ++ +-- +ID + + +type: keyword + +-- + +*`azure.auditlogs.properties.target_resources.type`*:: ++ +-- +Type + + +type: keyword + +-- + +*`azure.auditlogs.properties.target_resources.ip_address`*:: ++ +-- +ip Address + + +type: keyword + +-- + +*`azure.auditlogs.properties.target_resources.user_principal_name`*:: ++ +-- +User principal name + + +type: keyword + +-- + +[float] +=== modified_properties + +Modified properties + + + +*`azure.auditlogs.properties.target_resources.modified_properties.newValue`*:: ++ +-- +New value + + +type: keyword + +-- + +*`azure.auditlogs.properties.target_resources.modified_properties.displayName`*:: ++ +-- +Display value + + +type: keyword + +-- + +*`azure.auditlogs.properties.target_resources.modified_properties.oldValue`*:: ++ +-- +Old value + + +type: keyword + +-- + +[float] +=== initiated_by + +Information regarding the initiator + + + +[float] +=== app + +App + + + +*`azure.auditlogs.properties.initiated_by.app.servicePrincipalName`*:: ++ +-- +Service principal name + + +type: keyword + +-- + +*`azure.auditlogs.properties.initiated_by.app.displayName`*:: ++ +-- +Display name + + +type: keyword + +-- + +*`azure.auditlogs.properties.initiated_by.app.appId`*:: ++ +-- +App ID + + +type: keyword + +-- + +*`azure.auditlogs.properties.initiated_by.app.servicePrincipalId`*:: ++ +-- +Service principal ID + + +type: keyword + +-- + +[float] +=== user + +User + + + +*`azure.auditlogs.properties.initiated_by.user.userPrincipalName`*:: ++ +-- +User principal name + + +type: keyword + +-- + +*`azure.auditlogs.properties.initiated_by.user.displayName`*:: ++ +-- +Display name + + +type: keyword + +-- + +*`azure.auditlogs.properties.initiated_by.user.id`*:: ++ +-- +ID + + +type: keyword + +-- + +*`azure.auditlogs.properties.initiated_by.user.ipAddress`*:: ++ +-- +ip Address + + +type: keyword + +-- + +[float] +=== signinlogs + +Fields for Azure sign-in logs. + + + +*`azure.signinlogs.operation_name`*:: ++ +-- +The operation name + + +type: keyword + +-- + +*`azure.signinlogs.operation_version`*:: ++ +-- +The operation version + + +type: keyword + +-- + +*`azure.signinlogs.tenant_id`*:: ++ +-- +Tenant ID + + +type: keyword + +-- + +*`azure.signinlogs.result_signature`*:: ++ +-- +Result signature + + +type: keyword + +-- + +*`azure.signinlogs.correlation_id`*:: ++ +-- +Correlation ID + + +type: keyword + +-- + +*`azure.signinlogs.identity`*:: ++ +-- +Identity + + +type: keyword + +-- + +[float] +=== properties + +The signin log properties + + + +*`azure.signinlogs.properties.id`*:: ++ +-- +ID + + +type: keyword + +-- + +*`azure.signinlogs.properties.created_at`*:: ++ +-- +Created date time + + +type: keyword + +-- + +*`azure.signinlogs.properties.user_display_name`*:: ++ +-- +User display name + + +type: keyword + +-- + +*`azure.signinlogs.properties.correlation_id`*:: ++ +-- +Correlation ID + + +type: keyword + +-- + +*`azure.signinlogs.properties.user_principal_name`*:: ++ +-- +User principal name + + +type: keyword + +-- + +*`azure.signinlogs.properties.user_id`*:: ++ +-- +User ID + + +type: keyword + +-- + +*`azure.signinlogs.properties.app_id`*:: ++ +-- +App ID + + +type: keyword + +-- + +*`azure.signinlogs.properties.app_display_name`*:: ++ +-- +App display name + + +type: keyword + +-- + +*`azure.signinlogs.properties.ip_address`*:: ++ +-- +Ip address + + +type: keyword + +-- + +*`azure.signinlogs.properties.client_app_used`*:: ++ +-- +Client app used + + +type: keyword + +-- + +*`azure.signinlogs.properties.conditional_access_status`*:: ++ +-- +Conditional access status + + +type: keyword + +-- + +*`azure.signinlogs.properties.original_request_id`*:: ++ +-- +Original request ID + + +type: keyword + +-- + +*`azure.signinlogs.properties.is_interactive`*:: ++ +-- +Is interactive + + +type: keyword + +-- + +*`azure.signinlogs.properties.token_issuer_name`*:: ++ +-- +Token issuer name + + +type: keyword + +-- + +*`azure.signinlogs.properties.token_issuer_type`*:: ++ +-- +Token issuer type + + +type: keyword + +-- + +*`azure.signinlogs.properties.processing_time_ms`*:: ++ +-- +Processing time in milliseconds + + +type: float + +-- + +*`azure.signinlogs.properties.risk_detail`*:: ++ +-- +Risk detail + + +type: keyword + +-- + +*`azure.signinlogs.properties.risk_level_aggregated`*:: ++ +-- +Risk level aggregated + + +type: keyword + +-- + +*`azure.signinlogs.properties.risk_level_during_signin`*:: ++ +-- +Risk level during signIn + + +type: keyword + +-- + +*`azure.signinlogs.properties.risk_state`*:: ++ +-- +Risk state + + +type: keyword + +-- + +*`azure.signinlogs.properties.resource_display_name`*:: ++ +-- +Resource display name + + +type: keyword + +-- + +[float] +=== status + +Status + + + +*`azure.signinlogs.properties.status.error_code`*:: ++ +-- +Error code + + +type: keyword + +-- + +*`azure.signinlogs.properties.status.additional_details`*:: ++ +-- +Additional details + + +type: keyword + +-- + +[float] +=== device_detail + +Status + + + +*`azure.signinlogs.properties.device_detail.device_id`*:: ++ +-- +Device ID + + +type: keyword + +-- + +*`azure.signinlogs.properties.device_detail.operating_system`*:: ++ +-- +Operating system + + +type: keyword + +-- + +*`azure.signinlogs.properties.device_detail.browser`*:: ++ +-- +Browser + + +type: keyword + +-- + +*`azure.signinlogs.properties.device_detail.display_name`*:: ++ +-- +Display name + + +type: keyword + +-- + +*`azure.signinlogs.properties.device_detail.trust_type`*:: ++ +-- +Trust type + + +type: keyword + +-- + +*`azure.signinlogs.properties.service_principal_id`*:: ++ +-- +Status + + type: keyword -- diff --git a/filebeat/docs/modules/azure.asciidoc b/filebeat/docs/modules/azure.asciidoc new file mode 100644 index 00000000000..769196265ca --- /dev/null +++ b/filebeat/docs/modules/azure.asciidoc @@ -0,0 +1,110 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[filebeat-module-azure]] +[role="xpack"] + +:modulename: azure +:has-dashboards: false + +== azure module + +beta[] + +This is the azure module. + +The azure module will concentrate on retrieving different types of log data from Azure. +There are several requirements before using the module since the logs will actually be read from azure event hubs. + + - the event hubs the azure module will read logs from must have the kafka option enabled . + - the logs have to be exported first to the event hubs https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-create-kafka-enabled + - to export activity logs to event hubs users can follow the steps here https://docs.microsoft.com/en-us/azure/azure-monitor/platform/activity-log-export + - to export audit and sign-in logs to event hubs users can follow the steps here https://docs.microsoft.com/en-us/azure/active-directory/reports-monitoring/tutorial-azure-monitor-stream-logs-to-event-hub + +The module will contain the following filesets: + +`activitylogs` :: +Will retrieve azure activity logs. Control-plane events on Azure Resource Manager resources. Activity logs provide insight into the operations that were performed on resources in your subscription. + +`signinlogs` :: +Will retrieve azure Active Directory sign-in logs. The sign-ins report provides information about the usage of managed applications and user sign-in activities. + +`auditlogs` :: +Will retrieve azure Active Directory audit logs. The audit logs provide traceability through logs for all changes done by various features within Azure AD. Examples of audit logs include changes made to any resources within Azure AD like adding or removing users, apps, groups, roles and policies. + + +[float] +=== Module configuration + +``` +- module: azure + activitylogs: + enabled: true + var: + namespace: "obseventhubs.servicebus.windows.net:9093" + eventhub: ["insights-operational-logs"] + consumer_group: "$Default" + connection_string: "" + auditlogs: + enabled: true + var: + namespace: "" + eventhub: ["insights-logs-auditlogs"] + consumer_group: "$Default" + connection_string: "" + + signinlogs: + enabled: true + var: + namespace: "" + eventhub: ["insights-logs-signinlogs"] + consumer_group: "$Default" + connection_string: "" + +``` + + +A side by side kafka/event hubs notation, we will follow Azure notations in this case. + + +`namespace` :: +_string_ +An Event Hubs namespace provides a unique scoping container, referenced by its fully qualified domain name, in which users can create one or more event hubs or Kafka topics. + +`eventhub` :: + _[]string_ +Or kafka topic, is a fully managed, real-time data ingestion service. +Default value `insights-operational-logs` + +`consumer_group` :: +_string_ + The publish/subscribe mechanism of Event Hubs is enabled through consumer groups. A consumer group is a view (state, position, or offset) of an entire event hub. Consumer groups enable multiple consuming applications to each have a separate view of the event stream, and to read the stream independently at their own pace and with their own offsets. +Default value: `$Default` + +`connection_string` :: +_string_ +The connection string required to communicate with Event Hubs, steps here https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string. + + + +include::../include/what-happens.asciidoc[] + +[float] +=== Compatibility + +TODO: document with what versions of the software is this tested + + + + + + + + +[float] +=== Fields + +For a description of each field in the module, see the +<> section. + diff --git a/filebeat/docs/modules_list.asciidoc b/filebeat/docs/modules_list.asciidoc index ffee384c988..76b71d4deb0 100644 --- a/filebeat/docs/modules_list.asciidoc +++ b/filebeat/docs/modules_list.asciidoc @@ -6,6 +6,7 @@ This file is generated! See scripts/docs_collector.py * <> * <> * <> + * <> * <> * <> * <> @@ -44,6 +45,7 @@ include::modules-overview.asciidoc[] include::modules/apache.asciidoc[] include::modules/auditd.asciidoc[] include::modules/aws.asciidoc[] +include::modules/azure.asciidoc[] include::modules/cef.asciidoc[] include::modules/cisco.asciidoc[] include::modules/coredns.asciidoc[] diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index e5ea6d19f16..979d7b66972 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -87,6 +87,37 @@ filebeat.modules: # Profile name for aws credential #var.credential_profile_name: fb-aws +#-------------------------------- Azure Module -------------------------------- +- module: azure + # All logs + activitylogs: + enabled: true + var: + # Azure event hub namespace FQDN for example "eventhubs.servicebus.windows.net:9093" + namespace: "" + # Eventhub name containing the activity logs, overwrite he default value if the logs are exported in a different eventhub + eventhub: ["insights-operational-logs"] + # Consumer group name that has access to the event hub, we advise creating a dedicated consumer group for the azure module + consumer_group: "$Default" + # the connection string required to communicate with Event Hubs, steps to generate one here https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string + connection_string: "" + + # auditlogs: + # enabled: true + # var: + # namespace: "" + # eventhub: ["insights-logs-auditlogs"] + # consumer_group: "$Default" + # connection_string: "" + + # signinlogs: + # enabled: true + # var: + # namespace: "" + # eventhub: ["insights-logs-signinlogs"] + # consumer_group: "$Default" + # connection_string: "" + #--------------------------------- CEF Module --------------------------------- - module: cef log: diff --git a/x-pack/filebeat/include/list.go b/x-pack/filebeat/include/list.go index 93fde4fb469..4145e7601d4 100644 --- a/x-pack/filebeat/include/list.go +++ b/x-pack/filebeat/include/list.go @@ -13,6 +13,7 @@ import ( _ "github.com/elastic/beats/x-pack/filebeat/input/netflow" _ "github.com/elastic/beats/x-pack/filebeat/input/s3" _ "github.com/elastic/beats/x-pack/filebeat/module/aws" + _ "github.com/elastic/beats/x-pack/filebeat/module/azure" _ "github.com/elastic/beats/x-pack/filebeat/module/cef" _ "github.com/elastic/beats/x-pack/filebeat/module/cisco" _ "github.com/elastic/beats/x-pack/filebeat/module/coredns" diff --git a/x-pack/filebeat/module/azure/_meta/config.yml b/x-pack/filebeat/module/azure/_meta/config.yml new file mode 100644 index 00000000000..baaccbfd3b8 --- /dev/null +++ b/x-pack/filebeat/module/azure/_meta/config.yml @@ -0,0 +1,29 @@ +- module: azure + # All logs + activitylogs: + enabled: true + var: + # Azure event hub namespace FQDN for example "eventhubs.servicebus.windows.net:9093" + namespace: "" + # Eventhub name containing the activity logs, overwrite he default value if the logs are exported in a different eventhub + eventhub: ["insights-operational-logs"] + # Consumer group name that has access to the event hub, we advise creating a dedicated consumer group for the azure module + consumer_group: "$Default" + # the connection string required to communicate with Event Hubs, steps to generate one here https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string + connection_string: "" + + # auditlogs: + # enabled: true + # var: + # namespace: "" + # eventhub: ["insights-logs-auditlogs"] + # consumer_group: "$Default" + # connection_string: "" + + # signinlogs: + # enabled: true + # var: + # namespace: "" + # eventhub: ["insights-logs-signinlogs"] + # consumer_group: "$Default" + # connection_string: "" diff --git a/x-pack/filebeat/module/azure/_meta/docs.asciidoc b/x-pack/filebeat/module/azure/_meta/docs.asciidoc new file mode 100644 index 00000000000..68013c1f248 --- /dev/null +++ b/x-pack/filebeat/module/azure/_meta/docs.asciidoc @@ -0,0 +1,97 @@ +[role="xpack"] + +:modulename: azure +:has-dashboards: false + +== azure module + +beta[] + +This is the azure module. + +The azure module will concentrate on retrieving different types of log data from Azure. +There are several requirements before using the module since the logs will actually be read from azure event hubs. + + - the event hubs the azure module will read logs from must have the kafka option enabled . + - the logs have to be exported first to the event hubs https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-create-kafka-enabled + - to export activity logs to event hubs users can follow the steps here https://docs.microsoft.com/en-us/azure/azure-monitor/platform/activity-log-export + - to export audit and sign-in logs to event hubs users can follow the steps here https://docs.microsoft.com/en-us/azure/active-directory/reports-monitoring/tutorial-azure-monitor-stream-logs-to-event-hub + +The module will contain the following filesets: + +`activitylogs` :: +Will retrieve azure activity logs. Control-plane events on Azure Resource Manager resources. Activity logs provide insight into the operations that were performed on resources in your subscription. + +`signinlogs` :: +Will retrieve azure Active Directory sign-in logs. The sign-ins report provides information about the usage of managed applications and user sign-in activities. + +`auditlogs` :: +Will retrieve azure Active Directory audit logs. The audit logs provide traceability through logs for all changes done by various features within Azure AD. Examples of audit logs include changes made to any resources within Azure AD like adding or removing users, apps, groups, roles and policies. + + +[float] +=== Module configuration + +``` +- module: azure + activitylogs: + enabled: true + var: + namespace: "obseventhubs.servicebus.windows.net:9093" + eventhub: ["insights-operational-logs"] + consumer_group: "$Default" + connection_string: "" + auditlogs: + enabled: true + var: + namespace: "" + eventhub: ["insights-logs-auditlogs"] + consumer_group: "$Default" + connection_string: "" + + signinlogs: + enabled: true + var: + namespace: "" + eventhub: ["insights-logs-signinlogs"] + consumer_group: "$Default" + connection_string: "" + +``` + + +A side by side kafka/event hubs notation, we will follow Azure notations in this case. + + +`namespace` :: +_string_ +An Event Hubs namespace provides a unique scoping container, referenced by its fully qualified domain name, in which users can create one or more event hubs or Kafka topics. + +`eventhub` :: + _[]string_ +Or kafka topic, is a fully managed, real-time data ingestion service. +Default value `insights-operational-logs` + +`consumer_group` :: +_string_ + The publish/subscribe mechanism of Event Hubs is enabled through consumer groups. A consumer group is a view (state, position, or offset) of an entire event hub. Consumer groups enable multiple consuming applications to each have a separate view of the event stream, and to read the stream independently at their own pace and with their own offsets. +Default value: `$Default` + +`connection_string` :: +_string_ +The connection string required to communicate with Event Hubs, steps here https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string. + + + +include::../include/what-happens.asciidoc[] + +[float] +=== Compatibility + +TODO: document with what versions of the software is this tested + + + + + + diff --git a/x-pack/filebeat/module/azure/_meta/fields.yml b/x-pack/filebeat/module/azure/_meta/fields.yml new file mode 100644 index 00000000000..26050a7727f --- /dev/null +++ b/x-pack/filebeat/module/azure/_meta/fields.yml @@ -0,0 +1,14 @@ +- key: azure + title: "Azure" + release: beta + description: > + Azure Module + fields: + - name: azure + type: group + description: > + fields: + - name: resource_id + type: keyword + description: > + Resource ID diff --git a/x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml b/x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml new file mode 100644 index 00000000000..7d4ca2d0f6f --- /dev/null +++ b/x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml @@ -0,0 +1,79 @@ +- name: activitylogs + type: group + release: beta + description: > + Fields for Azure activity logs. + fields: + - name: identity + type: group + description: > + The canonical user ID of the owner of the source bucket. + fields: + - name: authorization + type: group + description: > + Node allocatable pods + fields: + - name: evidence + type: group + description: > + Node allocatable pods + fields: + - name: role_assignment_scope + type: keyword + description: > + Role assignment scope + - name: role_definition_id + type: keyword + description: > + Role definition ID + - name: role + type: keyword + description: > + Role + - name: role_assignment_id + type: keyword + description: > + Role assignment ID + - name: principal_id + type: keyword + description: > + Principal ID + - name: principal_type + type: keyword + description: > + Principal type + - name: scope + type: keyword + description: > + Scope + - name: action + type: keyword + description: > + Action + - name: correlation_id + type: keyword + description: > + Correlation ID + - name: operation_name + type: keyword + description: > + Operation name + - name: result_signature + type: keyword + description: > + Result signature + - name: properties + type: group + description: > + Properties + fields: + - name: service_request_id + type: keyword + description: > + Service Request Id + - name: status_code + type: keyword + description: > + Status code + diff --git a/x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml b/x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml new file mode 100644 index 00000000000..c22e189eb94 --- /dev/null +++ b/x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml @@ -0,0 +1,9 @@ +type: kafka +hosts: {{ .namespace }} +topics: {{ .eventhub }} +group_id: {{ .consumer_group }} +expand_event_list_from_field: "records" + +username: "$ConnectionString" +password: {{ .connection_string }} +ssl.enabled: true diff --git a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json new file mode 100644 index 00000000000..2bc11ee7d3d --- /dev/null +++ b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json @@ -0,0 +1,175 @@ +{ + "description": "Pipeline for parsing azure activity logs.", + "processors": [ + { + "json" : { + "field" : "message", + "target_field" : "azure.activitylogs" + } + }, + { + "drop": { + "if" : "ctx.azure.activitylogs.identity == null", + "ignore_failure": false + } + }, + { + "date": { + "field": "azure.activitylogs.time", + "target_field": "@timestamp", + "ignore_failure": false, + "formats": [ + "ISO8601" + ] + } + }, + { + "remove": { + "field": ["message", "azure.activitylogs.time"] + } + }, + { + "rename": { + "field": "azure.activitylogs.resourceId", + "target_field": "azure.resource_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.activitylogs.callerIpAddress", + "target_field": "source.ip", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.activitylogs.level", + "target_field": "log.level", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.activitylogs.durationMs", + "target_field": "event.duration", + "ignore_missing": true + } + }, + { + "script": { + "lang": "painless", + "source": "ctx.event.duration = ctx.event.duration * params.param_nano", + "params": { + "param_nano": 1000000 + } + } + }, + { + "rename": { + "field": "azure.activitylogs.location", + "target_field": "geo.name", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.activitylogs.category", + "target_field": "event.category", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.activitylogs.resultType", + "target_field": "event.outcome", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.activitylogs.operationName", + "target_field": "azure.activitylogs.operation_name", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.activitylogs.resultSignature", + "target_field": "azure.activitylogs.result_signature", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.activitylogs.identity.authorization.evidence.roleAssignmentScope", + "target_field": "azure.activitylogs.identity.authorization.evidence.role_assignment_scope", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.activitylogs.identity.authorization.evidence.roleDefinitionId", + "target_field": "azure.activitylogs.identity.authorization.evidence.role_definition_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.activitylogs.identity.authorization.evidence.roleAssignmentId", + "target_field": "azure.activitylogs.identity.authorization.evidence.role_assignment_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.activitylogs.identity.authorization.evidence.principalId", + "target_field": "azure.activitylogs.identity.authorization.evidence.principal_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.activitylogs.identity.authorization.evidence.principalType", + "target_field": "azure.activitylogs.identity.authorization.evidence.principal_type", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.activitylogs.correlationId", + "target_field": "azure.activitylogs.correlation_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.activitylogs.properties.serviceRequestId", + "target_field": "azure.activitylogs.properties.service_request_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.activitylogs.properties.statusMessage", + "target_field": "message", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.activitylogs.properties.statusCode", + "target_field": "azure.activitylogs.properties.status_code", + "ignore_missing": true + } + } + ], + "on_failure": [ + { + "set": { + "field": "error.message", + "value": "{{ _ingest.on_failure_message }}" + } + } + ] +} diff --git a/x-pack/filebeat/module/azure/activitylogs/manifest.yml b/x-pack/filebeat/module/azure/activitylogs/manifest.yml new file mode 100644 index 00000000000..d2bf93f2ce5 --- /dev/null +++ b/x-pack/filebeat/module/azure/activitylogs/manifest.yml @@ -0,0 +1,12 @@ +module_version: 1.0 + +var: + - name: input + default: kafka + - name: topics + default: "insights-operational-logs" + - name: consumer_group + default: "$Default" + +ingest_pipeline: ingest/pipeline.json +input: config/activitylogs.yml diff --git a/x-pack/filebeat/module/azure/activitylogs/test/activity_log.json b/x-pack/filebeat/module/azure/activitylogs/test/activity_log.json new file mode 100644 index 00000000000..2e4b6674c2c --- /dev/null +++ b/x-pack/filebeat/module/azure/activitylogs/test/activity_log.json @@ -0,0 +1,95 @@ +{ + "records": [ + { + "time": "2019-10-04T10:38:37.5151977Z", + "resourceId": "/SUBSCRIPTIONS/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/RESOURCEGROUPS/SA-HEMANT/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/AZURELSEVENTS/AUTHORIZATIONRULES/ROOTMANAGESHAREDACCESSKEY", + "operationName": "MICROSOFT.EVENTHUB/NAMESPACES/AUTHORIZATIONRULES/LISTKEYS/ACTION", + "category": "Action", + "resultType": "Start", + "resultSignature": "Started.", + "durationMs": 0, + "callerIpAddress": "51.191.161.11", + "correlationId": "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "identity": { + "authorization": { + "scope": "/subscriptions/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/resourceGroups/sa-hemant/providers/Microsoft.EventHub/namespaces/azurelsevents/authorizationRules/RootManageSharedAccessKey", + "action": "Microsoft.EventHub/namespaces/authorizationRules/listKeys/action", + "evidence": { + "role": "Azure EventGrid Service BuiltIn Role", + "roleAssignmentScope": "/subscriptions/2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "roleAssignmentId": "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "roleDefinitionId": "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "principalId": "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "principalType": "ServicePrincipal" + } + }, + "claims": { + "aud": "https://management.core.windows.net/", + "iss": "https://sts.windows.net/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/", + "iat": "1570185217", + "nbf": "1570185217", + "exp": "1570214317", + "aio": "2a7e2503-d7e2", + "appid": "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "appidacr": "2", + "http://schemas.microsoft.com/identity/claims/identityprovider": "https://sts.windows.net/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/", + "http://schemas.microsoft.com/identity/claims/objectidentifier": "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "http://schemas.xmlsoap.org/ws/2005/05/identity/claims/nameidentifier": "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "http://schemas.microsoft.com/identity/claims/tenantid": "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "uti": "2a7e2503", + "ver": "1.0" + } + }, + "level": "Information", + "location": "global" + }, + { + "time": "2019-10-04T10:38:37.5251981Z", + "resourceId": "/SUBSCRIPTIONS/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/RESOURCEGROUPS/SA-HEMANT/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/AZURELSEVENTS/AUTHORIZATIONRULES/ROOTMANAGESHAREDACCESSKEY", + "operationName": "MICROSOFT.EVENTHUB/NAMESPACES/AUTHORIZATIONRULES/LISTKEYS/ACTION", + "category": "Action", + "resultType": "Failure", + "resultSignature": "Failed.NotFound", + "durationMs": 11, + "callerIpAddress": "51.191.161.11", + "correlationId": "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "identity": { + "authorization": { + "scope": "/subscriptions/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/resourceGroups/sa-hemant/providers/Microsoft.EventHub/namespaces/azurelsevents/authorizationRules/RootManageSharedAccessKey", + "action": "Microsoft.EventHub/namespaces/authorizationRules/listKeys/action", + "evidence": { + "role": "Azure EventGrid Service BuiltIn Role", + "roleAssignmentScope": "/subscriptions/2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "roleAssignmentId": "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "roleDefinitionId": "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "principalId": "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "principalType": "ServicePrincipal" + } + }, + "claims": { + "aud": "https://management.core.windows.net/", + "iss": "https://sts.windows.net/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/", + "iat": "1570185217", + "nbf": "1570185217", + "exp": "1570214317", + "aio": "2a7e2503", + "appid": "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "appidacr": "2", + "http://schemas.microsoft.com/identity/claims/identityprovider": "https://sts.windows.net/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/", + "http://schemas.microsoft.com/identity/claims/objectidentifier": "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "http://schemas.xmlsoap.org/ws/2005/05/identity/claims/nameidentifier": "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "http://schemas.microsoft.com/identity/claims/tenantid": "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "uti": "2a7e2503", + "ver": "1.0" + } + }, + "level": "Error", + "location": "global", + "properties": { + "statusCode": "NotFound", + "serviceRequestId": null, + "statusMessage": "{\"error\":{\"code\":\"ParentResourceNotFound\",\"message\":\"Can not perform requested operation on nested resource. Parent resource 'azurelsevents' not found.\"}}" + } + } + ] +} diff --git a/x-pack/filebeat/module/azure/activitylogs/test/activity_log_expected.json b/x-pack/filebeat/module/azure/activitylogs/test/activity_log_expected.json new file mode 100644 index 00000000000..b1e06f171dc --- /dev/null +++ b/x-pack/filebeat/module/azure/activitylogs/test/activity_log_expected.json @@ -0,0 +1,113 @@ +{ + "_index" : "filebeat-8.0.0-2019.10.09", + "_type" : "_doc", + "_id" : "2a7e2503-d7e2-405a", + "_score" : null, + "_source" : { + "agent" : { + "hostname" : "DESKTOP-RFOOE09", + "id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "type" : "filebeat", + "ephemeral_id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "version" : "8.0.0" + }, + "log" : { + "level" : "Error" + }, + "source" : { + "ip" : "11.71.231.51" + }, + "fileset" : { + "name" : "activitylogs" + }, + "message" : """{"error":{"code":"ParentResourceNotFound","message":"Can not perform requested operation on nested resource. Parent resource 'azurelsevents' not found."}}""", + "geo" : { + "name" : "global" + }, + "input" : { + "type" : "kafka" + }, + "@timestamp" : "2019-10-09T19:39:22.822Z", + "ecs" : { + "version" : "1.1.0" + }, + "service" : { + "type" : "azure" + }, + "kafka" : { + "headers" : [ + "ProfileName: �\fname=default" + ], + "partition" : 3, + "offset" : 47915, + "topic" : "insights-operational-logs", + "key" : "" + }, + "host" : { + "hostname" : "DESKTOP-RFOOE09", + "os" : { + "build" : "18362.388", + "kernel" : "10.0.18362.388 (WinBuild.160101.0800)", + "name" : "Windows 10 Pro", + "family" : "windows", + "version" : "10.0", + "platform" : "windows" + }, + "name" : "DESKTOP-RFOOE09", + "id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "architecture" : "x86_64" + }, + "event" : { + "duration" : 1.1E7, + "module" : "azure", + "category" : "Action", + "dataset" : "azure.activitylogs", + "outcome" : "Failure" + }, + "azure" : { + "resource_id" : "/SUBSCRIPTIONS/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/RESOURCEGROUPS/SA-HEMANT/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/AZURELSEVENTS/AUTHORIZATIONRULES/ROOTMANAGESHAREDACCESSKEY", + "activitylogs" : { + "result_signature" : "Failed.NotFound", + "operation_name" : "MICROSOFT.EVENTHUB/NAMESPACES/AUTHORIZATIONRULES/LISTKEYS/ACTION", + "identity" : { + "authorization" : { + "evidence" : { + "role_definition_id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb7", + "role" : "Azure EventGrid Service BuiltIn Role", + "role_assignment_scope" : "/subscriptions/2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "role_assignment_id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "principal_type" : "ServicePrincipal", + "principal_id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73" + }, + "scope" : "/subscriptions/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/resourceGroups/sa-hemant/providers/Microsoft.EventHub/namespaces/azurelsevents/authorizationRules/RootManageSharedAccessKey", + "action" : "Microsoft.EventHub/namespaces/authorizationRules/listKeys/action" + }, + "claims" : { + "http://schemas.xmlsoap.org/ws/2005/05/identity/claims/nameidentifier" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "ver" : "1.0", + "http://schemas.microsoft.com/identity/claims/identityprovider" : "https://sts.windows.net/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/", + "aio" : "42VgYChmaFjYVOd58PiHOdfi963fDAA=", + "iss" : "https://sts.windows.net/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/", + "uti" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "aud" : "https://management.core.windows.net/", + "nbf" : "1570649662", + "appidacr" : "2", + "http://schemas.microsoft.com/identity/claims/tenantid" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "appid" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "http://schemas.microsoft.com/identity/claims/objectidentifier" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "exp" : "1570678762", + "iat" : "1570649662" + } + }, + "correlation_id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "properties" : { + "status_code" : "NotFound", + "service_request_id" : null + } + } + } + }, + "sort" : [ + 1570649962822 + ] +} diff --git a/x-pack/filebeat/module/azure/auditlogs/_meta/fields.yml b/x-pack/filebeat/module/azure/auditlogs/_meta/fields.yml new file mode 100644 index 00000000000..2df841208e5 --- /dev/null +++ b/x-pack/filebeat/module/azure/auditlogs/_meta/fields.yml @@ -0,0 +1,159 @@ +- name: auditlogs + type: group + description: > + Fields for Azure audit logs. + fields: + - name: operation_name + type: keyword + description: > + The operation name + - name: operation_version + type: keyword + description: > + The operation version + - name: tenant_id + type: keyword + description: > + Tenant ID + - name: result_signature + type: keyword + description: > + Result signature + - name: correlation_id + type: keyword + description: > + Correlation ID + - name: properties + type: group + description: > + The audit log properties + fields: + - name: result + type: keyword + description: > + Log result + - name: activity_display_name + type: keyword + description: > + Activity display name + - name: result_reason + type: keyword + description: > + Reason for the log result + - name: correlation_id + type: keyword + description: > + Correlation ID + - name: logged_by_service + type: keyword + description: > + Logged by service + - name: operation_type + type: keyword + description: > + Operation type + - name: id + type: keyword + description: > + ID + - name: activityDateTime + type: keyword + description: > + Activity timestamp + - name: category + type: keyword + description: > + category + - name: target_resources + type: group + description: > + Target resources + fields: + - name: display_name + type: keyword + description: > + Display name + - name: id + type: keyword + description: > + ID + - name: type + type: keyword + description: > + Type + - name: ip_address + type: keyword + description: > + ip Address + - name: user_principal_name + type: keyword + description: > + User principal name + - name: modified_properties + type: group + description: > + Modified properties + fields: + - name: newValue + type: keyword + description: > + New value + - name: displayName + type: keyword + description: > + Display value + - name: oldValue + type: keyword + description: > + Old value + - name: initiated_by + type: group + description: > + Information regarding the initiator + fields: + - name: app + type: group + description: > + App + fields: + - name: servicePrincipalName + type: keyword + description: > + Service principal name + - name: displayName + type: keyword + description: > + Display name + - name: appId + type: keyword + description: > + App ID + - name: servicePrincipalId + type: keyword + description: > + Service principal ID + - name: user + type: group + description: > + User + fields: + - name: userPrincipalName + type: keyword + description: > + User principal name + - name: displayName + type: keyword + description: > + Display name + - name: id + type: keyword + description: > + ID + - name: ipAddress + type: keyword + description: > + ip Address + + + + diff --git a/x-pack/filebeat/module/azure/auditlogs/config/auditlogs.yml b/x-pack/filebeat/module/azure/auditlogs/config/auditlogs.yml new file mode 100644 index 00000000000..c22e189eb94 --- /dev/null +++ b/x-pack/filebeat/module/azure/auditlogs/config/auditlogs.yml @@ -0,0 +1,9 @@ +type: kafka +hosts: {{ .namespace }} +topics: {{ .eventhub }} +group_id: {{ .consumer_group }} +expand_event_list_from_field: "records" + +username: "$ConnectionString" +password: {{ .connection_string }} +ssl.enabled: true diff --git a/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.json b/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.json new file mode 100644 index 00000000000..d8826e03011 --- /dev/null +++ b/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.json @@ -0,0 +1,196 @@ +{ + "description": "Pipeline for parsing azure activity logs.", + "processors": [ + { + "json" : { + "field" : "message", + "target_field" : "azure.auditlogs" + } + }, + { + "drop": { + "if" : "ctx.azure.auditlogs.category != 'AuditLogs'" + } + }, + { + "date": { + "field": "azure.auditlogs.time", + "target_field": "@timestamp", + "ignore_failure": false, + "formats": [ + "ISO8601" + ] + } + }, + { + "rename": { + "field": "azure.auditlogs.resourceId", + "target_field": "azure.resource_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.auditlogs.durationMs", + "target_field": "event.duration", + "ignore_missing": true + } + }, + { + "script": { + "lang": "painless", + "source": "ctx.event.duration = ctx.event.duration * params.param_nano", + "params": { + "param_nano": 1000000 + } + } + }, + { + "rename": { + "field": "azure.auditlogs.properties.result", + "target_field": "event.outcome", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.auditlogs.level", + "target_field": "log.level", + "ignore_missing": true + } + }, + { + "remove": { + "field": ["message", "azure.auditlogs.time"], + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.auditlogs.category", + "target_field": "event.category", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.auditlogs.operationName", + "target_field": "azure.auditlogs.operation_name", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.auditlogs.resultSignature", + "target_field": "azure.auditlogs.result_signature", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.auditlogs.operationVersion", + "target_field": "azure.auditlogs.operation_version", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.auditlogs.tenantId", + "target_field": "azure.auditlogs.tenant_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.auditlogs.correlationId", + "target_field": "azure.auditlogs.correlation_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.auditlogs.properties.activityDisplayName", + "target_field": "azure.auditlogs.properties.activity_display_name", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.auditlogs.properties.resultReason", + "target_field": "azure.auditlogs.properties.result_reason", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.auditlogs.properties.correlationId", + "target_field": "azure.auditlogs.properties.correlation_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.auditlogs.properties.loggedByService", + "target_field": "azure.auditlogs.properties.logged_by_service", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.auditlogs.properties.operationType", + "target_field": "azure.auditlogs.properties.operation_type", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.auditlogs.properties.targetResources", + "target_field": "azure.auditlogs.properties.target_resources", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.auditlogs.properties.target_resources.displayName", + "target_field": "azure.auditlogs.properties.target_resources.display_name", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.auditlogs.properties.target_resources.ipAddress", + "target_field": "azure.auditlogs.properties.target_resources.ip_address", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.auditlogs.properties.target_resources.userPrincipalName", + "target_field": "azure.auditlogs.properties.target_resources.user_principal_name", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.auditlogs.properties.target_resources.modifiedProperties", + "target_field": "azure.auditlogs.properties.target_resources.modified_properties", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.auditlogs.properties.initiatedBy", + "target_field": "azure.auditlogs.properties.initiated_by", + "ignore_missing": true + } + } + ], + "on_failure": [ + { + "set": { + "field": "error.message", + "value": "{{ _ingest.on_failure_message }}" + } + } + ] +} diff --git a/x-pack/filebeat/module/azure/auditlogs/manifest.yml b/x-pack/filebeat/module/azure/auditlogs/manifest.yml new file mode 100644 index 00000000000..f74e0f3593e --- /dev/null +++ b/x-pack/filebeat/module/azure/auditlogs/manifest.yml @@ -0,0 +1,12 @@ +module_version: 1.0 + +var: + - name: input + default: kafka + - name: eventhub + default: "insights-logs-auditlogs" + - name: consumer_group + default: "$Default" + +ingest_pipeline: ingest/pipeline.json +input: config/auditlogs.yml diff --git a/x-pack/filebeat/module/azure/auditlogs/test/audit_log.json b/x-pack/filebeat/module/azure/auditlogs/test/audit_log.json new file mode 100644 index 00000000000..7d0f57c6afb --- /dev/null +++ b/x-pack/filebeat/module/azure/auditlogs/test/audit_log.json @@ -0,0 +1,78 @@ +{ + "records": [ + { + "time":"2019-10-04T16:36:19.0359773Z", + "resourceId":"/tenants/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/providers/Microsoft.aadiam", + "operationName":"Add user", + "operationVersion":"1.0", + "category":"AuditLogs", + "tenantId":"2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "resultSignature":"None", + "durationMs":0, + "correlationId":"2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "level":"Informational", + "properties":{ + "id":"Directory_T2a7e2503-d7e2", + "category":"UserManagement", + "correlationId":"2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "result":"success", + "resultReason":"", + "activityDisplayName":"Add user", + "activityDateTime":"2019-10-04T16:36:19.0359773+00:00", + "loggedByService":"Core Directory", + "operationType":"Add", + "initiatedBy":{ + "user":{ + "id":"2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "displayName":null, + "userPrincipalName":"admin@elastic365.onmicrosoft.com", + "ipAddress":null + } + }, + "targetResources":[ + { + "id":"2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "displayName":null, + "type":"User", + "userPrincipalName":"test_testd@elastic.co", + "modifiedProperties":[ + { + "displayName":"AccountEnabled", + "oldValue":"[]", + "newValue":"[true]" + }, + { + "displayName":"PasswordPolicies", + "oldValue":"[]", + "newValue":"[\"None\"]" + }, + { + "displayName":"SourceAnchor", + "oldValue":"[]", + "newValue":"[\"2a7e2503\"]" + }, + { + "displayName":"UserPrincipalName", + "oldValue":"[]", + "newValue":"[\"test_testd@elastic.co\"]" + }, + { + "displayName":"UserType", + "oldValue":"[]", + "newValue":"[\"Member\"]" + }, + { + "displayName":"Included Updated Properties", + "oldValue":null, + "newValue":"\"AccountEnabled, PasswordPolicies, SourceAnchor, UserPrincipalName, UserType\"" + } + ] + } + ], + "additionalDetails":[ + + ] + } + } + ] +} diff --git a/x-pack/filebeat/module/azure/auditlogs/test/audit_log_expected.json b/x-pack/filebeat/module/azure/auditlogs/test/audit_log_expected.json new file mode 100644 index 00000000000..54c46b04d1a --- /dev/null +++ b/x-pack/filebeat/module/azure/auditlogs/test/audit_log_expected.json @@ -0,0 +1,124 @@ +{ + "_index" : "filebeat-8.0.0-2019.10.10", + "_type" : "_doc", + "_id" : "2a7e2503-d7e2-405", + "_score" : null, + "_source" : { + "agent" : { + "hostname" : "DESKTOP-RFOOE09", + "id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "type" : "filebeat", + "ephemeral_id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "version" : "8.0.0" + }, + "log" : { + "level" : "Informational" + }, + "fileset" : { + "name" : "auditlogs" + }, + "input" : { + "type" : "kafka" + }, + "@timestamp" : "2019-10-10T07:57:40.301Z", + "ecs" : { + "version" : "1.1.0" + }, + "service" : { + "type" : "azure" + }, + "kafka" : { + "headers" : [ ], + "partition" : 2, + "offset" : 48, + "topic" : "insights-logs-auditlogs", + "key" : "" + }, + "host" : { + "hostname" : "DESKTOP-RFOOE09", + "os" : { + "build" : "18362.388", + "kernel" : "10.0.18362.388 (WinBuild.160101.0800)", + "name" : "Windows 10 Pro", + "family" : "windows", + "version" : "10.0", + "platform" : "windows" + }, + "name" : "DESKTOP-RFOOE09", + "id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "architecture" : "x86_64" + }, + "event" : { + "duration" : 0.0, + "module" : "azure", + "category" : "AuditLogs", + "dataset" : "azure.auditlogs", + "outcome" : "success" + }, + "azure" : { + "auditlogs" : { + "operation_name" : "Add member to group", + "tenant_id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "operation_version" : "1.0", + "identity" : "Microsoft Office 365 Portal", + "correlation_id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "result_signature" : "None", + "properties" : { + "logged_by_service" : "Core Directory", + "initiated_by" : { + "user" : { + "displayName" : "Microsoft Office 365 Portal", + "ipAddress" : null, + "id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "userPrincipalName" : "test.test@elastic.co" + } + }, + "activity_display_name" : "Add member to group", + "operation_type" : "Assign", + "correlation_id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "activityDateTime" : "2019-10-10T07:57:40.3016161+00:00", + "id" : "Directory_TOE2090", + "target_resources" : [ + { + "displayName" : null, + "modifiedProperties" : [ + { + "newValue" : """"2a7e2503-d7e2-405a-a84c-c333b9f7cb73"""", + "displayName" : "Group.ObjectID", + "oldValue" : null + }, + { + "newValue" : """"cloud-developers"""", + "displayName" : "Group.DisplayName", + "oldValue" : null + }, + { + "newValue" : null, + "displayName" : "Group.WellKnownObjectName", + "oldValue" : null + } + ], + "id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "type" : "User", + "userPrincipalName" : "test.test@elastic.co" + }, + { + "groupType" : "azureAD", + "displayName" : null, + "modifiedProperties" : [ ], + "id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "type" : "Group" + } + ], + "additionalDetails" : [ ], + "category" : "GroupManagement", + "result_reason" : "" + } + }, + "resource_id" : "/tenants/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/providers/Microsoft.aadiam" + } + }, + "sort" : [ + 1570694260301 + ] +} diff --git a/x-pack/filebeat/module/azure/fields.go b/x-pack/filebeat/module/azure/fields.go new file mode 100644 index 00000000000..808e17f6395 --- /dev/null +++ b/x-pack/filebeat/module/azure/fields.go @@ -0,0 +1,23 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. + +package azure + +import ( + "github.com/elastic/beats/libbeat/asset" +) + +func init() { + if err := asset.SetFields("filebeat", "azure", asset.ModuleFieldsPri, AssetAzure); err != nil { + panic(err) + } +} + +// AssetAzure returns asset data. +// This is the base64 encoded gzipped contents of module/azure. +func AssetAzure() string { + return "eJzsW91u2zgTvc9TDHrfPkAuPsBfswsY2DaFm+6tQItjlTBNaknKgfv0C1I/liyRYhLRzgLVVRwLc47IOcPRIf0R9ni6B/KrUngHYJjheA8fVvbzhzsAhRyJxnvYoiF3ABR1rlhpmBT38L87AAB3L3yRtOI2xI4hp/reffURBDngOby9zKnEeyiUrMrmPxMxh2H6oRRqWakcM0a779qgezw9S9X//2To+to0gWD9MAIhuWFHZk5cFnqE0qdur8shisD+0z0c7KRqRq8FBIv4qXfv5TD0WTKKwjBzGnzpYznDyF5PPxFyIqRgOeFQaVSwfgC5A/MTQT4LVO2HZuS2Vb5H8+ki0BTlweBW5qdU7BexPEZ3hfhHPIO9vkqKQDiXOTFkyxFKSfXEnT6ifbJ4tKOc4+RNc1wj+b6E8xzvPnclOWZEa1aIAwqT6VyWvgeBgIhe+Uj22kiOcKYAYQoD4hR3TDALMpT6tVif8fsVIkT5+iRfnAI3Gcne/EeMZKmYyFlJ+NXJfmuRX0bTMrkV0UnslqJPbHOMIph8nwzdXz4Dpf1NyKtx7BY2l0ohJ9MlIwQ9A/v5HPcyM1poWaKqge3n5aAf27gwittriCpuMisxYs5t1gLgGxcZpiOfdWCf3TC8XK1e3YZ880Wcayw0qiPLMVP4T4XaU+yWSP4aBzY1DqzHcTpKhphKZ7mkqYToAGAAMG5nK8pMVC/7op7VRn1Rw5pKJbZxlfNKOcMfUelxhVqMwVT4loRBQaYW4reAu5C+0nTj+nDDorx8abLT3KW9P/5coaqnJE1B+EsWvviXb7cZZbrk5DSlxoXYrNr32gZqLM7xuGQKiU7VPmxcbFfF7Issnx+tYAIvxCqQzH0qXBYF0mx7ypqlLlkKFUhhewIfzLiieprgBdicW6Bgs5tqagLT0erogRh8YsklZNgBtSGHsdPQpSoxWEh1aQgtRMQbvVvaiCrQyre2h6bsizd7O08OA0IYMbbOTOWDyPe2yHe2h1Dxg7kkXpiM5922m0T/2+yCJJ58KN1IlBmhVKH2mWALkgFWwioA1nKqNKrs/N5/ndz5oVGdzYb5FDpIynYMaebtToYkF/AuvzSQ/oaovWKdS4HPfxNeXdlW+YrPcAzCXpSPr/4MSESxrSRxNCWnNxjGR049/DpxC2YYMa6VSbFIrMVOqkPdNCgsiKJMFK7fa5CleuW6QUqfVhZT08oLEauepmvrXMLrp2nrkETULXhPsopiScpywvFJym9VljGe9OXEX5vmeNpnmg27oqbW0w8/RqygLM0bqim2BYD/nJSuvcMToSFWhlrBZMwmWtCYPzrls0IwsbjFa8N+ZOK3yTvB4LfJ+y5M3uAhmFeBrqcipvWUa/m+zVS+gQGWK3RdPEnkZX+u4wMlBp395WXiXs3T29luJY6yst+RaxxnWyw1OjN9yoBUqoH5UR9d84KTskyG7WnV+9BX2HYpy7g0DdpsSxSPEnzxO6VwhsJkdmAqjamk4kDs6MMkyFm2grozX4RnJM9R66zewk+l4A4OajjwwHXtjWIFs+xSH3J4bIBAtacc/EnNdMaEQeU2RBKl9FpDCKPrvOQeRca0rlAlFNiThYEaJqywAaF0u2QDQsGNslJJm2hMFJldUrODP7d3XE4u7RF8vnUobuEGJuDAOGcarcT8+a2Y3mcUDWE8zUBtmN6DB2BAguMReUaKQmFhu5CEdBwUBKAmiNFK2SmsO8fk3Go016eux2gDeraAJUpyR2g6/ujnAekX2e4HBFEr7cwy8ga3+7svctT5dqWk8p1Pg2W3sv6wWODF6lok2i3BtU6vsAW46jDBh9k5a+gOGc7UqFvNZ8PuGhvJDw5qzuJtDBNbqk7a4CE9r8cWEQKILb2tks9zJvQirP4fAHrnpxKMqrQJHXNf8mCAxZo51t4c8535ocBih2u7r+7+DQAA//8dk8dD" +} diff --git a/x-pack/filebeat/module/azure/module.yml b/x-pack/filebeat/module/azure/module.yml new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/x-pack/filebeat/module/azure/module.yml @@ -0,0 +1 @@ + diff --git a/x-pack/filebeat/module/azure/signinlogs/_meta/fields.yml b/x-pack/filebeat/module/azure/signinlogs/_meta/fields.yml new file mode 100644 index 00000000000..12b580e73cc --- /dev/null +++ b/x-pack/filebeat/module/azure/signinlogs/_meta/fields.yml @@ -0,0 +1,161 @@ +- name: signinlogs + type: group + description: > + Fields for Azure sign-in logs. + fields: + - name: operation_name + type: keyword + description: > + The operation name + - name: operation_version + type: keyword + description: > + The operation version + - name: tenant_id + type: keyword + description: > + Tenant ID + - name: result_signature + type: keyword + description: > + Result signature + - name: correlation_id + type: keyword + description: > + Correlation ID + - name: identity + type: keyword + description: > + Identity + - name: properties + type: group + description: > + The signin log properties + fields: + - name: id + type: keyword + description: > + ID + - name: created_at + type: keyword + description: > + Created date time + - name: user_display_name + type: keyword + description: > + User display name + - name: correlation_id + type: keyword + description: > + Correlation ID + - name: user_principal_name + type: keyword + description: > + User principal name + - name: user_id + type: keyword + description: > + User ID + - name: app_id + type: keyword + description: > + App ID + - name: app_display_name + type: keyword + description: > + App display name + - name: ip_address + type: keyword + description: > + Ip address + - name: client_app_used + type: keyword + description: > + Client app used + - name: conditional_access_status + type: keyword + description: > + Conditional access status + - name: original_request_id + type: keyword + description: > + Original request ID + - name: is_interactive + type: keyword + description: > + Is interactive + - name: token_issuer_name + type: keyword + description: > + Token issuer name + - name: token_issuer_type + type: keyword + description: > + Token issuer type + - name: processing_time_ms + type: float + description: > + Processing time in milliseconds + - name: risk_detail + type: keyword + description: > + Risk detail + - name: risk_level_aggregated + type: keyword + description: > + Risk level aggregated + - name: risk_level_during_signin + type: keyword + description: > + Risk level during signIn + - name: risk_state + type: keyword + description: > + Risk state + - name: resource_display_name + type: keyword + description: > + Resource display name + - name: status + type: group + description: > + Status + fields: + - name: error_code + type: keyword + description: > + Error code + - name: additional_details + type: keyword + description: > + Additional details + - name: device_detail + type: group + description: > + Status + fields: + - name: device_id + type: keyword + description: > + Device ID + - name: operating_system + type: keyword + description: > + Operating system + - name: browser + type: keyword + description: > + Browser + - name: display_name + type: keyword + description: > + Display name + - name: trust_type + type: keyword + description: > + Trust type + - name: service_principal_id + type: keyword + description: > + Status + diff --git a/x-pack/filebeat/module/azure/signinlogs/config/signinlogs.yml b/x-pack/filebeat/module/azure/signinlogs/config/signinlogs.yml new file mode 100644 index 00000000000..c22e189eb94 --- /dev/null +++ b/x-pack/filebeat/module/azure/signinlogs/config/signinlogs.yml @@ -0,0 +1,9 @@ +type: kafka +hosts: {{ .namespace }} +topics: {{ .eventhub }} +group_id: {{ .consumer_group }} +expand_event_list_from_field: "records" + +username: "$ConnectionString" +password: {{ .connection_string }} +ssl.enabled: true diff --git a/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.json b/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.json new file mode 100644 index 00000000000..589b8ba1560 --- /dev/null +++ b/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.json @@ -0,0 +1,378 @@ +{ + "description": "Pipeline for parsing azure signin logs.", + "processors": [ + { + "json" : { + "field" : "message", + "target_field" : "azure.signinlogs" + } + }, + { + "drop": { + "if" : "ctx.azure.signinlogs.category != 'SignInLogs'" + } + }, + { + "date": { + "field": "azure.signinlogs.time", + "target_field": "@timestamp", + "ignore_failure": false, + "formats": [ + "ISO8601" + ] + } + }, + { + "rename": { + "field": "azure.signinlogs.resourceId", + "target_field": "azure.resource_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.callerIpAddress", + "target_field": "source.ip", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.Level", + "target_field": "log.level", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.durationMs", + "target_field": "event.duration", + "ignore_missing": true + } + }, + { + "script": { + "lang": "painless", + "source": "ctx.event.duration = ctx.event.duration * params.param_nano", + "params": { + "param_nano": 1000000 + } + } + }, + { + "rename": { + "field": "azure.signinlogs.location", + "target_field": "geo.country_iso_code", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.resultType", + "target_field": "event.outcome", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.category", + "target_field": "event.category", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.operationName", + "target_field": "azure.signinlogs.operation_name", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.resultSignature", + "target_field": "azure.signinlogs.result_signature", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.operationVersion", + "target_field": "azure.signinlogs.operation_version", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.tenantId", + "target_field": "azure.signinlogs.tenant_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.correlationId", + "target_field": "azure.signinlogs.correlation_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.networkLocationDetails", + "target_field": "azure.signinlogs.properties.network_location_details", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.resourceId", + "target_field": "azure.signinlogs.properties.resource_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.appliedConditionalAccessPolicies", + "target_field": "azure.signinlogs.properties.applied_conditional_access_policies", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.deviceDetail", + "target_field": "azure.signinlogs.properties.device_detail", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.device_detail.deviceId", + "target_field": "azure.signinlogs.properties.device_detail.device_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.device_detail.operatingSystem", + "target_field": "azure.signinlogs.properties.device_detail.operating_system", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.device_detail.displayName", + "target_field": "azure.signinlogs.properties.device_detail.display_name", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.device_detail.trustType", + "target_field": "azure.signinlogs.properties.device_detail.trust_type", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.createdDateTime", + "target_field": "azure.signinlogs.properties.created_at", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.userDisplayName", + "target_field": "azure.signinlogs.properties.user_display_name", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.correlationId", + "target_field": "azure.signinlogs.properties.correlation_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.userPrincipalName", + "target_field": "azure.signinlogs.properties.user_principal_name", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.userId", + "target_field": "azure.signinlogs.properties.user_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.appId", + "target_field": "azure.signinlogs.properties.app_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.appDisplayName", + "target_field": "azure.signinlogs.properties.app_display_name", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.ipAddress", + "target_field": "azure.signinlogs.properties.ip_address", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.clientAppUsed", + "target_field": "azure.signinlogs.properties.client_app_used", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.conditionalAccessStatus", + "target_field": "azure.signinlogs.properties.conditional_access_status", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.originalRequestId", + "target_field": "azure.signinlogs.properties.original_request_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.isInteractive", + "target_field": "azure.signinlogs.properties.is_interactive", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.tokenIssuerName", + "target_field": "azure.signinlogs.properties.token_issuer_name", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.tokenIssuerType", + "target_field": "azure.signinlogs.properties.token_issuer_type", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.processingTimeInMilliseconds", + "target_field": "azure.signinlogs.properties.processing_time_ms", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.riskDetail", + "target_field": "azure.signinlogs.properties.risk_detail", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.riskLevelAggregated", + "target_field": "azure.signinlogs.properties.risk_level_aggregated", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.riskLevelDuringSignIn", + "target_field": "azure.signinlogs.properties.risk_level_during_signin", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.riskState", + "target_field": "azure.signinlogs.properties.risk_state", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.resourceDisplayName", + "target_field": "azure.signinlogs.properties.resource_display_name", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.status.errorCode", + "target_field": "azure.signinlogs.properties.status.error_code", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.status.additionalDetails", + "target_field": "azure.signinlogs.properties.status.additional_details", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.location.city", + "target_field": "geo.city_name", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.location.state", + "target_field": "geo.country_name", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.location.geoCoordinates.latitude", + "target_field": "geo.location.lat", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.location.geoCoordinates.longitude", + "target_field": "geo.location.lon", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.servicePrincipalId", + "target_field": "azure.signinlogs.properties.service_principal_id", + "ignore_missing": true + } + }, + { + "remove": { + "field": ["message", "azure.signinlogs.time","azure.signinlogs.properties.location"], + "ignore_missing": true + } + } + ], + "on_failure": [ + { + "set": { + "field": "error.message", + "value": "{{ _ingest.on_failure_message }}" + } + } + ] +} diff --git a/x-pack/filebeat/module/azure/signinlogs/manifest.yml b/x-pack/filebeat/module/azure/signinlogs/manifest.yml new file mode 100644 index 00000000000..785bc7e01c1 --- /dev/null +++ b/x-pack/filebeat/module/azure/signinlogs/manifest.yml @@ -0,0 +1,12 @@ +module_version: 1.0 + +var: + - name: input + default: kafka + - name: eventhub + default: "insights-logs-auditlogs" + - name: consumer_group + default: "$Default" + +ingest_pipeline: ingest/pipeline.json +input: config/signinlogs.yml diff --git a/x-pack/filebeat/module/azure/signinlogs/test/signin_log.json b/x-pack/filebeat/module/azure/signinlogs/test/signin_log.json new file mode 100644 index 00000000000..f2c07d16193 --- /dev/null +++ b/x-pack/filebeat/module/azure/signinlogs/test/signin_log.json @@ -0,0 +1,155 @@ +{ + "time":"2019-10-04T10:47:34.5949655Z", + "resourceId":"/tenants/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/providers/Microsoft.aadiam", + "operationName":"Sign-in activity", + "operationVersion":"1.0", + "category":"SignInLogs", + "tenantId":"2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "resultType":"0", + "resultSignature":"None", + "durationMs":0, + "callerIpAddress":"81.191.171.1", + "correlationId":"2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "identity":"Test Test", + "Level":4, + "location":"GB", + "properties":{ + "SignInBondData":{ + "LocationDetails":{ + "Latitude":51.11111, + "Longitude":-2.1111, + "IPChain":null + }, + "MfaDetails":null, + "ConditionalAccessDetails":{ + "MultiCAEvaluationLog":"0|" + }, + "RamDetails":null, + "DeviceDetails":{ + "DeviceDisplayName":null, + "DeviceId":"", + "IsCompliant":null, + "IsManaged":null, + "DeviceTrustType":null, + "UserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36", + "DevicePlatform":"MacOs", + "BrowserId":"2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "BrowserType":"Chrome" + }, + "DisplayDetails":{ + "UserName":"Test Test", + "ApplicationDisplayName":"Microsoft Office Web Apps Service", + "ProxyRestrictionTargetTenantName":null, + "ResourceDisplayName":"Windows Azure Active Directory", + "AttemptedUsername":null + }, + "ProtocolDetails":{ + "ResponseTime":131, + "IsInteractive":null, + "AuthenticationMethodsUsed":null, + "NetworkLocation":null, + "DomainHintPresent":null, + "LoginHintPresent":null, + "Protocol":null + }, + "PassThroughAuthenticationDetails":null, + "IssuerDetails":null, + "SourceAlpEvents":null + }, + "id":"2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "createdDateTime":"2019-10-04T10:47:34.5949655+00:00", + "userDisplayName":"Test Test", + "userPrincipalName":"test_test@elastic.co", + "userId":"2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "appId":"2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "appDisplayName":"Microsoft Office Web Apps Service", + "ipAddress":"81.191.171.1", + "status":{ + "errorCode":0 + }, + "clientAppUsed":"Browser", + "deviceDetail":{ + "deviceId":"", + "operatingSystem":"MacOs", + "browser":"Chrome 73.0.3683" + }, + "location":{ + "city":"City1", + "state":"State1", + "countryOrRegion":"GB", + "geoCoordinates":{ + "latitude":51.111111111111, + "longitude":-2.1111111111111111111 + } + }, + "correlationId":"2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "conditionalAccessStatus":"notApplied", + "appliedConditionalAccessPolicies":[ + { + "id":"2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "displayName":"Require MFA for Azure", + "enforcedGrantControls":[ + "Mfa" + ], + "enforcedSessionControls":[ + + ], + "result":"notApplied", + "conditionsSatisfied":1, + "conditionsNotSatisfied":2 + }, + { + "id":"2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "displayName":"Disable Persistent Browser Sessions", + "enforcedGrantControls":[ + + ], + "enforcedSessionControls":[ + + ], + "result":"notEnabled", + "conditionsSatisfied":0, + "conditionsNotSatisfied":0 + }, + { + "id":"2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "displayName":"12-hour Session timeouts", + "enforcedGrantControls":[ + + ], + "enforcedSessionControls":[ + + ], + "result":"notEnabled", + "conditionsSatisfied":0, + "conditionsNotSatisfied":0 + } + ], + "originalRequestId":"2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "isInteractive":false, + "tokenIssuerName":"", + "tokenIssuerType":"AzureAD", + "authenticationProcessingDetails":[ + + ], + "networkLocationDetails":[ + + ], + "processingTimeInMilliseconds":131, + "riskDetail":"none", + "riskLevelAggregated":"none", + "riskLevelDuringSignIn":"none", + "riskState":"none", + "riskEventTypes":[ + + ], + "resourceDisplayName":"Windows Azure Active Directory", + "resourceId":"2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "authenticationDetails":[ + + ], + "authenticationRequirementPolicies":[ + + ] + } +} diff --git a/x-pack/filebeat/module/azure/signinlogs/test/signin_log_expected.json b/x-pack/filebeat/module/azure/signinlogs/test/signin_log_expected.json new file mode 100644 index 00000000000..49658b5e70b --- /dev/null +++ b/x-pack/filebeat/module/azure/signinlogs/test/signin_log_expected.json @@ -0,0 +1,127 @@ +{ + "_index" : "filebeat-8.0.0-2019.10.09", + "_type" : "_doc", + "_id" : "2a7e2503-d7e2-405a-a84c-c3", + "_score" : null, + "_source" : { + "agent" : { + "hostname" : "DESKTOP-RFOOE09", + "id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "type" : "filebeat", + "ephemeral_id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "version" : "8.0.0" + }, + "log" : { + "level" : 4 + }, + "source" : { + "ip" : "91.171.71.101" + }, + "fileset" : { + "name" : "signinlogs" + }, + "geo" : { + "city_name" : "Paris", + "country_iso_code" : "FR", + "country_name" : "Paris", + "location" : { + "lon" : 2.341434343434343, + "lat" : 48.24353454545454 + } + }, + "input" : { + "type" : "kafka" + }, + "@timestamp" : "2019-10-09T20:57:56.668Z", + "ecs" : { + "version" : "1.1.0" + }, + "service" : { + "type" : "azure" + }, + "kafka" : { + "headers" : [ ], + "partition" : 2, + "offset" : 1326, + "topic" : "insights-logs-signinlogs", + "key" : "" + }, + "host" : { + "hostname" : "DESKTOP-RFOOE09", + "os" : { + "build" : "18362.388", + "kernel" : "10.0.18362.388 (WinBuild.160101.0800)", + "name" : "Windows 10 Pro", + "family" : "windows", + "version" : "10.0", + "platform" : "windows" + }, + "name" : "DESKTOP-RFOOE09", + "id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "architecture" : "x86_64" + }, + "event" : { + "duration" : 0.0, + "module" : "azure", + "category" : "SignInLogs", + "dataset" : "azure.signinlogs", + "outcome" : "0" + }, + "azure" : { + "signinlogs" : { + "tenant_id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "operation_version" : "1.0", + "result_signature" : "None", + "operation_name" : "Sign-in activity", + "identity" : "Test Test", + "correlation_id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "properties" : { + "risk_level_aggregated" : "none", + "applied_conditional_access_policies" : [ ], + "created_at" : "2019-10-09T20:57:56.6687852+00:00", + "risk_level_during_signin" : "none", + "authenticationProcessingDetails" : [ + { + "value" : "True", + "key" : "Domain Hint Present" + } + ], + "token_issuer_type" : "AzureAD", + "original_request_id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "app_id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "authenticationDetails" : [ ], + "network_location_details" : [ ], + "riskEventTypes" : [ ], + "is_interactive" : false, + "service_principal_id" : "", + "authenticationRequirementPolicies" : [ ], + "app_display_name" : "Microsoft Teams Web Client", + "ip_address" : "91.171.71.101", + "device_detail" : { + "device_id" : "", + "browser" : "Chrome 77.0.3865", + "operating_system" : "MacOs" + }, + "risk_detail" : "none", + "token_issuer_name" : "", + "resource_display_name" : "Microsoft Teams Services", + "risk_state" : "none", + "user_principal_name" : "test@elastic.co", + "processing_time_ms" : 601, + "user_id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "resource_id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "correlation_id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "user_display_name" : "Test Test", + "status" : { + "error_code" : 0 + } + } + }, + "resource_id" : "/tenants/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/providers/Microsoft.aadiam" + } + }, + "sort" : [ + 1570654676668 + ] +} diff --git a/x-pack/filebeat/modules.d/azure.yml.disabled b/x-pack/filebeat/modules.d/azure.yml.disabled new file mode 100644 index 00000000000..61f827236cc --- /dev/null +++ b/x-pack/filebeat/modules.d/azure.yml.disabled @@ -0,0 +1,32 @@ +# Module: azure +# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-azure.html + +- module: azure + # All logs + activitylogs: + enabled: true + var: + # Azure event hub namespace FQDN for example "eventhubs.servicebus.windows.net:9093" + namespace: "" + # Eventhub name containing the activity logs, overwrite he default value if the logs are exported in a different eventhub + eventhub: ["insights-operational-logs"] + # Consumer group name that has access to the event hub, we advise creating a dedicated consumer group for the azure module + consumer_group: "$Default" + # the connection string required to communicate with Event Hubs, steps to generate one here https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string + connection_string: "" + + # auditlogs: + # enabled: true + # var: + # namespace: "" + # eventhub: ["insights-logs-auditlogs"] + # consumer_group: "$Default" + # connection_string: "" + + # signinlogs: + # enabled: true + # var: + # namespace: "" + # eventhub: ["insights-logs-signinlogs"] + # consumer_group: "$Default" + # connection_string: ""