From 736a73b5a10f5c4d93ca62993c308128e1aaea6d Mon Sep 17 00:00:00 2001 From: Mariana Date: Thu, 19 Sep 2019 12:17:12 +0200 Subject: [PATCH 01/21] First commit --- filebeat/docs/fields.asciidoc | 13 ++++ filebeat/docs/modules/azure.asciidoc | 60 +++++++++++++++++++ filebeat/docs/modules_list.asciidoc | 2 + x-pack/filebeat/filebeat.reference.yml | 10 ++++ x-pack/filebeat/include/list.go | 1 + x-pack/filebeat/module/azure/_meta/config.yml | 8 +++ .../filebeat/module/azure/_meta/docs.asciidoc | 47 +++++++++++++++ x-pack/filebeat/module/azure/_meta/fields.yml | 9 +++ x-pack/filebeat/module/azure/fields.go | 23 +++++++ x-pack/filebeat/module/azure/module.yml | 3 + .../module/azure/monitor/config/monitor.yml | 6 ++ .../module/azure/monitor/ingest/pipeline.json | 11 ++++ .../module/azure/monitor/manifest.yml | 13 ++++ x-pack/filebeat/modules.d/azure.yml.disabled | 11 ++++ 14 files changed, 217 insertions(+) create mode 100644 filebeat/docs/modules/azure.asciidoc create mode 100644 x-pack/filebeat/module/azure/_meta/config.yml create mode 100644 x-pack/filebeat/module/azure/_meta/docs.asciidoc create mode 100644 x-pack/filebeat/module/azure/_meta/fields.yml create mode 100644 x-pack/filebeat/module/azure/fields.go create mode 100644 x-pack/filebeat/module/azure/module.yml create mode 100644 x-pack/filebeat/module/azure/monitor/config/monitor.yml create mode 100644 x-pack/filebeat/module/azure/monitor/ingest/pipeline.json create mode 100644 x-pack/filebeat/module/azure/monitor/manifest.yml create mode 100644 x-pack/filebeat/modules.d/azure.yml.disabled diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index ac9b7ef3d05..573f52b903c 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -15,6 +15,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -1241,6 +1242,18 @@ type: keyword -- +[[exported-fields-azure]] +== azure fields + +azure Module + + + +[float] +=== azure + + + [[exported-fields-beat-common]] == Beat fields diff --git a/filebeat/docs/modules/azure.asciidoc b/filebeat/docs/modules/azure.asciidoc new file mode 100644 index 00000000000..3251b267668 --- /dev/null +++ b/filebeat/docs/modules/azure.asciidoc @@ -0,0 +1,60 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[filebeat-module-azure]] +:modulename: azure +:has-dashboards: true + +== azure module + +This is the azure module. + +include::../include/what-happens.asciidoc[] + +[float] +=== Compatibility + +TODO: document with what versions of the software is this tested + + +include::../include/running-modules.asciidoc[] + +[float] +=== Example dashboard + +This module comes with a sample dashboard. For example: + +TODO: include an image of a sample dashboard. If you do not include a dashboard, +remove this section and set `:has-dashboards: false` at the top of this file. + +include::../include/configuring-intro.asciidoc[] + +TODO: provide an example configuration + +:fileset_ex: {fileset} + +include::../include/config-option-intro.asciidoc[] + +TODO: document the variables from each fileset. If you're describing a variable +that's common to other modules, you can reuse shared descriptions by including +the relevant file. For example: + +[float] +==== `{fileset}` log fileset settings + +include::../include/var-paths.asciidoc[] + +:has-dashboards!: + +:fileset_ex!: + +:modulename!: + + +[float] +=== Fields + +For a description of each field in the module, see the +<> section. + diff --git a/filebeat/docs/modules_list.asciidoc b/filebeat/docs/modules_list.asciidoc index ffee384c988..76b71d4deb0 100644 --- a/filebeat/docs/modules_list.asciidoc +++ b/filebeat/docs/modules_list.asciidoc @@ -6,6 +6,7 @@ This file is generated! See scripts/docs_collector.py * <> * <> * <> + * <> * <> * <> * <> @@ -44,6 +45,7 @@ include::modules-overview.asciidoc[] include::modules/apache.asciidoc[] include::modules/auditd.asciidoc[] include::modules/aws.asciidoc[] +include::modules/azure.asciidoc[] include::modules/cef.asciidoc[] include::modules/cisco.asciidoc[] include::modules/coredns.asciidoc[] diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index e5ea6d19f16..c4d94446800 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -87,6 +87,16 @@ filebeat.modules: # Profile name for aws credential #var.credential_profile_name: fb-aws +#-------------------------------- Azure Module -------------------------------- +- module: azure + # All logs + {fileset}: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + #--------------------------------- CEF Module --------------------------------- - module: cef log: diff --git a/x-pack/filebeat/include/list.go b/x-pack/filebeat/include/list.go index 47ba991534f..401e4226c48 100644 --- a/x-pack/filebeat/include/list.go +++ b/x-pack/filebeat/include/list.go @@ -12,6 +12,7 @@ import ( _ "github.com/elastic/beats/x-pack/filebeat/input/netflow" _ "github.com/elastic/beats/x-pack/filebeat/input/s3" _ "github.com/elastic/beats/x-pack/filebeat/module/aws" + _ "github.com/elastic/beats/x-pack/filebeat/module/azure" _ "github.com/elastic/beats/x-pack/filebeat/module/cef" _ "github.com/elastic/beats/x-pack/filebeat/module/cisco" _ "github.com/elastic/beats/x-pack/filebeat/module/coredns" diff --git a/x-pack/filebeat/module/azure/_meta/config.yml b/x-pack/filebeat/module/azure/_meta/config.yml new file mode 100644 index 00000000000..93e4241714d --- /dev/null +++ b/x-pack/filebeat/module/azure/_meta/config.yml @@ -0,0 +1,8 @@ +- module: azure + # All logs + {fileset}: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/x-pack/filebeat/module/azure/_meta/docs.asciidoc b/x-pack/filebeat/module/azure/_meta/docs.asciidoc new file mode 100644 index 00000000000..6e68e7da183 --- /dev/null +++ b/x-pack/filebeat/module/azure/_meta/docs.asciidoc @@ -0,0 +1,47 @@ +:modulename: azure +:has-dashboards: true + +== azure module + +This is the azure module. + +include::../include/what-happens.asciidoc[] + +[float] +=== Compatibility + +TODO: document with what versions of the software is this tested + + +include::../include/running-modules.asciidoc[] + +[float] +=== Example dashboard + +This module comes with a sample dashboard. For example: + +TODO: include an image of a sample dashboard. If you do not include a dashboard, +remove this section and set `:has-dashboards: false` at the top of this file. + +include::../include/configuring-intro.asciidoc[] + +TODO: provide an example configuration + +:fileset_ex: {fileset} + +include::../include/config-option-intro.asciidoc[] + +TODO: document the variables from each fileset. If you're describing a variable +that's common to other modules, you can reuse shared descriptions by including +the relevant file. For example: + +[float] +==== `{fileset}` log fileset settings + +include::../include/var-paths.asciidoc[] + +:has-dashboards!: + +:fileset_ex!: + +:modulename!: diff --git a/x-pack/filebeat/module/azure/_meta/fields.yml b/x-pack/filebeat/module/azure/_meta/fields.yml new file mode 100644 index 00000000000..dc7c1477832 --- /dev/null +++ b/x-pack/filebeat/module/azure/_meta/fields.yml @@ -0,0 +1,9 @@ +- key: azure + title: "azure" + description: > + azure Module + fields: + - name: azure + type: group + description: > + fields: diff --git a/x-pack/filebeat/module/azure/fields.go b/x-pack/filebeat/module/azure/fields.go new file mode 100644 index 00000000000..5b5ef06eb79 --- /dev/null +++ b/x-pack/filebeat/module/azure/fields.go @@ -0,0 +1,23 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. + +package azure + +import ( + "github.com/elastic/beats/libbeat/asset" +) + +func init() { + if err := asset.SetFields("filebeat", "azure", asset.ModuleFieldsPri, AssetAzure); err != nil { + panic(err) + } +} + +// AssetAzure returns asset data. +// This is the base64 encoded gzipped contents of module/azure. +func AssetAzure() string { + return "eJxsjLsNAyEQRHOqGJHTwAbuwEUgs7ZWxoBgCXD1J7iTSG6y+T2HLw+C//fKBlDRyAS7vDVA4PaqUlRyIjwMgHOLZw49zstbOIZGq3JI/scbN6WjMOFTcy9XcsPcmCMAAP//fWgqmA==" +} diff --git a/x-pack/filebeat/module/azure/module.yml b/x-pack/filebeat/module/azure/module.yml new file mode 100644 index 00000000000..19cdc5b48e4 --- /dev/null +++ b/x-pack/filebeat/module/azure/module.yml @@ -0,0 +1,3 @@ +dashboards: +- id: Filebeat-azure-monitor-Dashboard + file: Filebeat-azure-monitor.json diff --git a/x-pack/filebeat/module/azure/monitor/config/monitor.yml b/x-pack/filebeat/module/azure/monitor/config/monitor.yml new file mode 100644 index 00000000000..0afd17317d4 --- /dev/null +++ b/x-pack/filebeat/module/azure/monitor/config/monitor.yml @@ -0,0 +1,6 @@ +type: log +paths: +{{ range $i, $path := .paths }} + - {{$path}} +{{ end }} +exclude_files: [".gz$"] diff --git a/x-pack/filebeat/module/azure/monitor/ingest/pipeline.json b/x-pack/filebeat/module/azure/monitor/ingest/pipeline.json new file mode 100644 index 00000000000..a9936bce1a1 --- /dev/null +++ b/x-pack/filebeat/module/azure/monitor/ingest/pipeline.json @@ -0,0 +1,11 @@ +{ + "description": "Pipeline for parsing azure monitor logs", + "processors": [ + ], + "on_failure" : [{ + "set" : { + "field" : "error.message", + "value" : "{{ _ingest.on_failure_message }}" + } + }] +} diff --git a/x-pack/filebeat/module/azure/monitor/manifest.yml b/x-pack/filebeat/module/azure/monitor/manifest.yml new file mode 100644 index 00000000000..a77d193cdd2 --- /dev/null +++ b/x-pack/filebeat/module/azure/monitor/manifest.yml @@ -0,0 +1,13 @@ +module_version: 1.0 + +var: + - name: paths + default: + - /example/test.log* + os.darwin: + - /usr/local/example/test.log* + os.windows: + - c:/programdata/example/logs/test.log* + +ingest_pipeline: ingest/pipeline.json +input: config/monitor.yml diff --git a/x-pack/filebeat/modules.d/azure.yml.disabled b/x-pack/filebeat/modules.d/azure.yml.disabled new file mode 100644 index 00000000000..61df0a4bf36 --- /dev/null +++ b/x-pack/filebeat/modules.d/azure.yml.disabled @@ -0,0 +1,11 @@ +# Module: azure +# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-azure.html + +- module: azure + # All logs + {fileset}: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: From d99085f00c14e59eb1ff6791b849dc8468ef8b47 Mon Sep 17 00:00:00 2001 From: Mariana Date: Fri, 20 Sep 2019 10:29:47 +0200 Subject: [PATCH 02/21] Creating the filebeat module --- x-pack/filebeat/module/azure/_meta/config.yml | 2 +- .../azure/activitylogs/config/activitylogs.yml | 8 ++++++++ .../{monitor => activitylogs}/ingest/pipeline.json | 2 +- .../filebeat/module/azure/activitylogs/manifest.yml | 7 +++++++ x-pack/filebeat/module/azure/module.yml | 4 ++-- .../module/azure/monitor/config/monitor.yml | 6 ------ x-pack/filebeat/module/azure/monitor/manifest.yml | 13 ------------- .../module/cisco/shared/gen-ecs-mapping-docs.go | 3 ++- 8 files changed, 21 insertions(+), 24 deletions(-) create mode 100644 x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml rename x-pack/filebeat/module/azure/{monitor => activitylogs}/ingest/pipeline.json (72%) create mode 100644 x-pack/filebeat/module/azure/activitylogs/manifest.yml delete mode 100644 x-pack/filebeat/module/azure/monitor/config/monitor.yml delete mode 100644 x-pack/filebeat/module/azure/monitor/manifest.yml diff --git a/x-pack/filebeat/module/azure/_meta/config.yml b/x-pack/filebeat/module/azure/_meta/config.yml index 93e4241714d..90e310e2e43 100644 --- a/x-pack/filebeat/module/azure/_meta/config.yml +++ b/x-pack/filebeat/module/azure/_meta/config.yml @@ -1,6 +1,6 @@ - module: azure # All logs - {fileset}: + activitylogs: enabled: true # Set custom paths for the log files. If left empty, diff --git a/x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml b/x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml new file mode 100644 index 00000000000..0939522e5d9 --- /dev/null +++ b/x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml @@ -0,0 +1,8 @@ +- type: kafka + hosts: ["obseventhuba.servicebus.windows.net:9093"] + topics: ["insights-operational-logs"] + group_id: "$Default" + + username: "$ConnectionString" + password: "Endpoint=sb://obseventhuba.servicebus.windows.net/;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=f0FlBLjw6/MFfMqm0RbRzgimkFmtOtHBsjBp+A97zwg=" + ssl.enabled: true diff --git a/x-pack/filebeat/module/azure/monitor/ingest/pipeline.json b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json similarity index 72% rename from x-pack/filebeat/module/azure/monitor/ingest/pipeline.json rename to x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json index a9936bce1a1..a12c4303843 100644 --- a/x-pack/filebeat/module/azure/monitor/ingest/pipeline.json +++ b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json @@ -1,5 +1,5 @@ { - "description": "Pipeline for parsing azure monitor logs", + "description": "Pipeline for parsing azure activity logs", "processors": [ ], "on_failure" : [{ diff --git a/x-pack/filebeat/module/azure/activitylogs/manifest.yml b/x-pack/filebeat/module/azure/activitylogs/manifest.yml new file mode 100644 index 00000000000..9c3ca39c1b2 --- /dev/null +++ b/x-pack/filebeat/module/azure/activitylogs/manifest.yml @@ -0,0 +1,7 @@ +module_version: 1.0 + +var: + - name: input + +ingest_pipeline: ingest/pipeline.json +input: config/activitylogs.yml diff --git a/x-pack/filebeat/module/azure/module.yml b/x-pack/filebeat/module/azure/module.yml index 19cdc5b48e4..3d473aed6c4 100644 --- a/x-pack/filebeat/module/azure/module.yml +++ b/x-pack/filebeat/module/azure/module.yml @@ -1,3 +1,3 @@ dashboards: -- id: Filebeat-azure-monitor-Dashboard - file: Filebeat-azure-monitor.json +- id: Filebeat-azure-activitylogs-Dashboard + file: Filebeat-azure-activitylogs.json diff --git a/x-pack/filebeat/module/azure/monitor/config/monitor.yml b/x-pack/filebeat/module/azure/monitor/config/monitor.yml deleted file mode 100644 index 0afd17317d4..00000000000 --- a/x-pack/filebeat/module/azure/monitor/config/monitor.yml +++ /dev/null @@ -1,6 +0,0 @@ -type: log -paths: -{{ range $i, $path := .paths }} - - {{$path}} -{{ end }} -exclude_files: [".gz$"] diff --git a/x-pack/filebeat/module/azure/monitor/manifest.yml b/x-pack/filebeat/module/azure/monitor/manifest.yml deleted file mode 100644 index a77d193cdd2..00000000000 --- a/x-pack/filebeat/module/azure/monitor/manifest.yml +++ /dev/null @@ -1,13 +0,0 @@ -module_version: 1.0 - -var: - - name: paths - default: - - /example/test.log* - os.darwin: - - /usr/local/example/test.log* - os.windows: - - c:/programdata/example/logs/test.log* - -ingest_pipeline: ingest/pipeline.json -input: config/monitor.yml diff --git a/x-pack/filebeat/module/cisco/shared/gen-ecs-mapping-docs.go b/x-pack/filebeat/module/cisco/shared/gen-ecs-mapping-docs.go index 62fc5f41914..7e5ed04bd8f 100644 --- a/x-pack/filebeat/module/cisco/shared/gen-ecs-mapping-docs.go +++ b/x-pack/filebeat/module/cisco/shared/gen-ecs-mapping-docs.go @@ -10,6 +10,7 @@ import ( "encoding/csv" "flag" "fmt" + "github.com/elastic/beats/libbeat/common" "io" "os" "sort" @@ -39,7 +40,7 @@ var outputTables = []struct { type idMappings map[string]fieldMappings -type fieldMappings map[string]stringSet +type fieldMappings map[string]common.StringSet func main() { if err := generate(); err != nil { From 6f6b6c538eb213412ac6d0355ed2d409631014c0 Mon Sep 17 00:00:00 2001 From: Mariana Date: Fri, 20 Sep 2019 15:17:11 +0200 Subject: [PATCH 03/21] Work on module --- .../filebeat/module/azure/activitylogs/_meta/fields.yml | 5 +++++ .../module/azure/activitylogs/config/activitylogs.yml | 8 -------- .../filebeat/module/azure/activitylogs/config/kafka.yml | 8 ++++++++ x-pack/filebeat/module/azure/activitylogs/manifest.yml | 5 +++-- x-pack/filebeat/modules.d/azure.yml.disabled | 2 +- 5 files changed, 17 insertions(+), 11 deletions(-) create mode 100644 x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml delete mode 100644 x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml create mode 100644 x-pack/filebeat/module/azure/activitylogs/config/kafka.yml diff --git a/x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml b/x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml new file mode 100644 index 00000000000..859a914d5a9 --- /dev/null +++ b/x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml @@ -0,0 +1,5 @@ +- name: activitylogs + type: group + description: > + Fields for azure Activity logs. + fields: diff --git a/x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml b/x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml deleted file mode 100644 index 0939522e5d9..00000000000 --- a/x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml +++ /dev/null @@ -1,8 +0,0 @@ -- type: kafka - hosts: ["obseventhuba.servicebus.windows.net:9093"] - topics: ["insights-operational-logs"] - group_id: "$Default" - - username: "$ConnectionString" - password: "Endpoint=sb://obseventhuba.servicebus.windows.net/;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=f0FlBLjw6/MFfMqm0RbRzgimkFmtOtHBsjBp+A97zwg=" - ssl.enabled: true diff --git a/x-pack/filebeat/module/azure/activitylogs/config/kafka.yml b/x-pack/filebeat/module/azure/activitylogs/config/kafka.yml new file mode 100644 index 00000000000..eae526c4476 --- /dev/null +++ b/x-pack/filebeat/module/azure/activitylogs/config/kafka.yml @@ -0,0 +1,8 @@ +type: kafka +hosts: ["obseventhuba.servicebus.windows.net:9093"] +topics: ["insights-operational-logs"] +group_id: "$Default" + +username: "$ConnectionString" +password: "Endpoint=sb://obseventhuba.servicebus.windows.net/;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=f0FlBLjw6/MFfMqm0RbRzgimkFmtOtHBsjBp+A97zwg=" +ssl.enabled: true diff --git a/x-pack/filebeat/module/azure/activitylogs/manifest.yml b/x-pack/filebeat/module/azure/activitylogs/manifest.yml index 9c3ca39c1b2..dc3a3390bd4 100644 --- a/x-pack/filebeat/module/azure/activitylogs/manifest.yml +++ b/x-pack/filebeat/module/azure/activitylogs/manifest.yml @@ -2,6 +2,7 @@ module_version: 1.0 var: - name: input + default: kafka -ingest_pipeline: ingest/pipeline.json -input: config/activitylogs.yml +ingest_pipeline: ingest/pipeline.yml +input: config/{{.input}}.yml diff --git a/x-pack/filebeat/modules.d/azure.yml.disabled b/x-pack/filebeat/modules.d/azure.yml.disabled index 61df0a4bf36..5f9c1de9bb4 100644 --- a/x-pack/filebeat/modules.d/azure.yml.disabled +++ b/x-pack/filebeat/modules.d/azure.yml.disabled @@ -3,7 +3,7 @@ - module: azure # All logs - {fileset}: + activitylogs: enabled: true # Set custom paths for the log files. If left empty, From 43992b5699740c663f17d0fad0631a77c5ccaa56 Mon Sep 17 00:00:00 2001 From: Mariana Date: Fri, 20 Sep 2019 17:38:04 +0200 Subject: [PATCH 04/21] Work on the azure module --- .../module/azure/activitylogs/config/kafka.yml | 4 ++-- .../module/azure/activitylogs/ingest/pipeline.json | 11 ----------- .../module/azure/activitylogs/ingest/pipeline.yml | 12 ++++++++++++ .../filebeat/module/azure/activitylogs/manifest.yml | 2 +- 4 files changed, 15 insertions(+), 14 deletions(-) delete mode 100644 x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json create mode 100644 x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml diff --git a/x-pack/filebeat/module/azure/activitylogs/config/kafka.yml b/x-pack/filebeat/module/azure/activitylogs/config/kafka.yml index eae526c4476..e36ba19655d 100644 --- a/x-pack/filebeat/module/azure/activitylogs/config/kafka.yml +++ b/x-pack/filebeat/module/azure/activitylogs/config/kafka.yml @@ -1,8 +1,8 @@ type: kafka -hosts: ["obseventhuba.servicebus.windows.net:9093"] +hosts: [""] topics: ["insights-operational-logs"] group_id: "$Default" username: "$ConnectionString" -password: "Endpoint=sb://obseventhuba.servicebus.windows.net/;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=f0FlBLjw6/MFfMqm0RbRzgimkFmtOtHBsjBp+A97zwg=" +password: "" ssl.enabled: true diff --git a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json deleted file mode 100644 index a12c4303843..00000000000 --- a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "description": "Pipeline for parsing azure activity logs", - "processors": [ - ], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml new file mode 100644 index 00000000000..abd0101e5d9 --- /dev/null +++ b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml @@ -0,0 +1,12 @@ +{ + "description": "Pipeline for parsing azure activity logs", + "processors": [], + "on_failure": [ + { + "set": { + "field": "error.message", + "value": "{{ _ingest.on_failure_message }}" + } + } + ] +} diff --git a/x-pack/filebeat/module/azure/activitylogs/manifest.yml b/x-pack/filebeat/module/azure/activitylogs/manifest.yml index dc3a3390bd4..7638f902468 100644 --- a/x-pack/filebeat/module/azure/activitylogs/manifest.yml +++ b/x-pack/filebeat/module/azure/activitylogs/manifest.yml @@ -5,4 +5,4 @@ var: default: kafka ingest_pipeline: ingest/pipeline.yml -input: config/{{.input}}.yml +input: config/kafka.yml From 9932a5b0e107b05d7f565843b6be619db6cf383d Mon Sep 17 00:00:00 2001 From: Mariana Date: Mon, 23 Sep 2019 17:14:54 +0200 Subject: [PATCH 05/21] Temp commit --- filebeat/docs/fields.asciidoc | 14 ++ x-pack/filebeat/filebeat.reference.yml | 2 +- x-pack/filebeat/filebeat.yml | 5 +- x-pack/filebeat/include/list.go | 1 + .../azure/activitylogs/config/kafka.yml | 7 +- .../azure/activitylogs/ingest/pipeline.json | 29 +++ .../azure/activitylogs/ingest/pipeline.yml | 12 -- .../module/azure/activitylogs/manifest.yml | 2 +- x-pack/filebeat/module/azure/fields.go | 2 +- .../{azure.yml.disabled => azure.yml} | 0 .../processors/decode_azure/_meta/fields.yml | 6 + .../processors/decode_azure/decode_azure.go | 201 ++++++++++++++++++ .../processors/decode_azure/fields.go | 23 ++ 13 files changed, 286 insertions(+), 18 deletions(-) create mode 100644 x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json delete mode 100644 x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml rename x-pack/filebeat/modules.d/{azure.yml.disabled => azure.yml} (100%) create mode 100644 x-pack/filebeat/processors/decode_azure/_meta/fields.yml create mode 100644 x-pack/filebeat/processors/decode_azure/decode_azure.go create mode 100644 x-pack/filebeat/processors/decode_azure/fields.go diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 573f52b903c..75cf6da0e8c 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -16,6 +16,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -1254,6 +1255,19 @@ azure Module + +[float] +=== activitylogs + +Fields for azure Activity logs. + + +[[exported-fields-azure]] +== Decode CEF processor fields fields + +Common Event Format (CEF) data. + + [[exported-fields-beat-common]] == Beat fields diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index c4d94446800..a6cc9c3019e 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -90,7 +90,7 @@ filebeat.modules: #-------------------------------- Azure Module -------------------------------- - module: azure # All logs - {fileset}: + activitylogs: enabled: true # Set custom paths for the log files. If left empty, diff --git a/x-pack/filebeat/filebeat.yml b/x-pack/filebeat/filebeat.yml index d02b4d161fb..16d99954455 100644 --- a/x-pack/filebeat/filebeat.yml +++ b/x-pack/filebeat/filebeat.yml @@ -143,7 +143,7 @@ setup.kibana: #================================ Outputs ===================================== # Configure what output to use when sending the data collected by the beat. - +filebeat.overwrite_pipelines: true #-------------------------- Elasticsearch output ------------------------------ output.elasticsearch: # Array of hosts to connect to. @@ -183,7 +183,8 @@ processors: # Sets log level. The default log level is info. # Available log levels are: error, warning, info, debug -#logging.level: debug +logging.level: debug +setup.ilm.enabled: false # At debug level, you can selectively enable logging only for some components. # To enable all selectors use ["*"]. Examples of other selectors are "beat", diff --git a/x-pack/filebeat/include/list.go b/x-pack/filebeat/include/list.go index 401e4226c48..ec269a10f06 100644 --- a/x-pack/filebeat/include/list.go +++ b/x-pack/filebeat/include/list.go @@ -26,5 +26,6 @@ import ( _ "github.com/elastic/beats/x-pack/filebeat/module/rabbitmq" _ "github.com/elastic/beats/x-pack/filebeat/module/suricata" _ "github.com/elastic/beats/x-pack/filebeat/module/zeek" + _ "github.com/elastic/beats/x-pack/filebeat/processors/decode_azure" _ "github.com/elastic/beats/x-pack/filebeat/processors/decode_cef" ) diff --git a/x-pack/filebeat/module/azure/activitylogs/config/kafka.yml b/x-pack/filebeat/module/azure/activitylogs/config/kafka.yml index e36ba19655d..b3d1e363423 100644 --- a/x-pack/filebeat/module/azure/activitylogs/config/kafka.yml +++ b/x-pack/filebeat/module/azure/activitylogs/config/kafka.yml @@ -1,8 +1,13 @@ type: kafka -hosts: [""] +hosts: ["{}.servicebus.windows.net:9093"] topics: ["insights-operational-logs"] group_id: "$Default" username: "$ConnectionString" password: "" ssl.enabled: true + +processors: +- decode_json_fields: + fields: ["message"] + target: "" diff --git a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json new file mode 100644 index 00000000000..97a86859079 --- /dev/null +++ b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json @@ -0,0 +1,29 @@ +{ + "description": "Pipeline for parsing azure activity logs.", + "processors": [ + { + "foreach": { + "field": "records", + "processor" : { + "uppercase" : { + "field" : "_ingest._value.category" + } + } + } + }, + { + "remove": { + "field":"message", + "ignore_missing": true + } + } + ], + "on_failure": [ + { + "set": { + "field": "error.message", + "value": "{{ _ingest.on_failure_message }}" + } + } + ] +} diff --git a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml deleted file mode 100644 index abd0101e5d9..00000000000 --- a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml +++ /dev/null @@ -1,12 +0,0 @@ -{ - "description": "Pipeline for parsing azure activity logs", - "processors": [], - "on_failure": [ - { - "set": { - "field": "error.message", - "value": "{{ _ingest.on_failure_message }}" - } - } - ] -} diff --git a/x-pack/filebeat/module/azure/activitylogs/manifest.yml b/x-pack/filebeat/module/azure/activitylogs/manifest.yml index 7638f902468..65d744199b1 100644 --- a/x-pack/filebeat/module/azure/activitylogs/manifest.yml +++ b/x-pack/filebeat/module/azure/activitylogs/manifest.yml @@ -4,5 +4,5 @@ var: - name: input default: kafka -ingest_pipeline: ingest/pipeline.yml +ingest_pipeline: ingest/pipeline.json input: config/kafka.yml diff --git a/x-pack/filebeat/module/azure/fields.go b/x-pack/filebeat/module/azure/fields.go index 5b5ef06eb79..65c676abc24 100644 --- a/x-pack/filebeat/module/azure/fields.go +++ b/x-pack/filebeat/module/azure/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAzure returns asset data. // This is the base64 encoded gzipped contents of module/azure. func AssetAzure() string { - return "eJxsjLsNAyEQRHOqGJHTwAbuwEUgs7ZWxoBgCXD1J7iTSG6y+T2HLw+C//fKBlDRyAS7vDVA4PaqUlRyIjwMgHOLZw49zstbOIZGq3JI/scbN6WjMOFTcy9XcsPcmCMAAP//fWgqmA==" + return "eJx0z00KwyAQBeC9p3hknx7ARaGb7nqIECdhqI2iY8GevmjsTyCZnb7nh9PjTlljeKVAChAWSxpdPXcKMBTHwF7YLRpnBWDt4uZMsuXJxGRN1DXqsQwP+nFlJHvSmINLvt3smFtmQ43CT5Zs3Ry/4Z56KK9zrT4mF9oClwajyKe/7ucn7wAAAP//cWNQHw==" } diff --git a/x-pack/filebeat/modules.d/azure.yml.disabled b/x-pack/filebeat/modules.d/azure.yml similarity index 100% rename from x-pack/filebeat/modules.d/azure.yml.disabled rename to x-pack/filebeat/modules.d/azure.yml diff --git a/x-pack/filebeat/processors/decode_azure/_meta/fields.yml b/x-pack/filebeat/processors/decode_azure/_meta/fields.yml new file mode 100644 index 00000000000..a979dcdb5f7 --- /dev/null +++ b/x-pack/filebeat/processors/decode_azure/_meta/fields.yml @@ -0,0 +1,6 @@ +- key: azure + title: Decode azure processor fields + description: > + Common Event Format (azure) data. + fields: + diff --git a/x-pack/filebeat/processors/decode_azure/decode_azure.go b/x-pack/filebeat/processors/decode_azure/decode_azure.go new file mode 100644 index 00000000000..c18c26a8cb8 --- /dev/null +++ b/x-pack/filebeat/processors/decode_azure/decode_azure.go @@ -0,0 +1,201 @@ +package decode_azure + +import ( + "encoding/json" + "github.com/elastic/beats/libbeat/beat" + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/common/cfgwarn" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/libbeat/processors" + "github.com/elastic/beats/x-pack/filebeat/processors/decode_cef/cef" + "github.com/pkg/errors" + "go.uber.org/multierr" + "strconv" + "strings" +) + +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + + + +const ( + procName = "decode_azure" + logName = "processor." + procName +) + +type config struct { + Field string `config:"field"` // Source field containing the CEF message. + TargetField string `config:"target_field"` // Target field for the CEF object. + IgnoreMissing bool `config:"ignore_missing"` // Ignore missing source field. + IgnoreFailure bool `config:"ignore_failure"` // Ignore failures when the source field does not contain a CEF message. Parse errors do not cause failures, but are added to error.message. + ID string `config:"id"` // Instance ID for debugging purposes. + ECS bool `config:"ecs"` // Generate ECS fields. +} + +func defaultConfig() config { + return config{ + Field: "message", + TargetField: "cef", + ECS: true, + } +} + +func init() { + processors.RegisterPlugin(procName, New) +} + +type processor struct { + config + log *logp.Logger +} + +// New constructs a new processor built from ucfg config. +func New(cfg *common.Config) (processors.Processor, error) { + c := defaultConfig() + if err := cfg.Unpack(&c); err != nil { + return nil, errors.Wrap(err, "fail to unpack the "+procName+" processor configuration") + } + + return newDecodeCEF(c) +} + +func newDecodeCEF(c config) (*processor, error) { + cfgwarn.Beta("The " + procName + " processor is a beta feature.") + + log := logp.NewLogger(logName) + if c.ID != "" { + log = log.With("instance_id", c.ID) + } + + return &processor{config: c, log: log}, nil +} + +func (p *processor) String() string { + json, _ := json.Marshal(p.config) + return procName + "=" + string(json) +} + +func (p *processor) Run(event *beat.Event) (*beat.Event, error) { + v, err := event.GetValue(p.Field) + if err != nil { + if p.IgnoreMissing { + return event, nil + } + return event, errors.Wrapf(err, "decode_cef field [%v] not found", p.Field) + } + + cefData, ok := v.(string) + if !ok { + if p.IgnoreFailure { + return event, nil + } + return event, errors.Wrapf(err, "decode_cef field [%v] is not a string", p.Field) + } + + // Ignore any leading data before the CEF header. + idx := strings.Index(cefData, "CEF:") + if idx == -1 { + if p.IgnoreFailure { + return event, nil + } + return event, errors.Errorf("decode_cef field [%v] does not contain a CEF header", p.Field) + } + cefData = cefData[idx:] + + // If the version < 0 after parsing then none of the data is valid so return here. + var ce cef.Event + if err = ce.Unpack([]byte(cefData), cef.WithFullExtensionNames()); ce.Version < 0 && err != nil { + if p.IgnoreFailure { + return event, nil + } + return event, errors.Wrap(err, "decode_cef failed to parse message") + } + + cefErrors := multierr.Errors(err) + cefObject := toCEFObject(&ce) + event.PutValue(p.TargetField, cefObject) + + + // Add all parsing/conversion errors to error.message. + for _, cefError := range cefErrors { + if err := appendErrorMessage(event.Fields, cefError.Error()); err != nil { + p.log.Warn("Failed adding CEF errors to event.", "error", err) + break + } + } + + return event, nil +} + +func toCEFObject(cefEvent *cef.Event) common.MapStr { + // Add CEF header fields. + cefObject := common.MapStr{"version": strconv.Itoa(cefEvent.Version)} + if cefEvent.DeviceVendor != "" { + cefObject.Put("device.vendor", cefEvent.DeviceVendor) + } + if cefEvent.DeviceProduct != "" { + cefObject.Put("device.product", cefEvent.DeviceProduct) + } + if cefEvent.DeviceVersion != "" { + cefObject.Put("device.version", cefEvent.DeviceVersion) + } + if cefEvent.DeviceEventClassID != "" { + cefObject.Put("device.event_class_id", cefEvent.DeviceEventClassID) + } + if cefEvent.Name != "" { + cefObject.Put("name", cefEvent.Name) + } + if cefEvent.Severity != "" { + cefObject.Put("severity", cefEvent.Severity) + } + + // Add CEF extensions (key-value pairs). + if len(cefEvent.Extensions) > 0 { + extensions := make(common.MapStr, len(cefEvent.Extensions)) + cefObject.Put("extensions", extensions) + for k, v := range cefEvent.Extensions { + extensions.Put(k, v) + } + } + + return cefObject +} + + + +func appendErrorMessage(m common.MapStr, msg string) error { + const field = "error.message" + list, _ := m.GetValue(field) + + switch v := list.(type) { + case nil: + m.Put(field, msg) + case string: + if msg != v { + m.Put(field, []string{v, msg}) + } + case []string: + for _, existingTag := range v { + if msg == existingTag { + // Duplicate + return nil + } + } + m.Put(field, append(v, msg)) + case []interface{}: + for _, existingTag := range v { + if msg == existingTag { + // Duplicate + return nil + } + } + m.Put(field, append(v, msg)) + default: + return errors.Errorf("unexpected type %T found for %v field", list, field) + } + return nil +} + + diff --git a/x-pack/filebeat/processors/decode_azure/fields.go b/x-pack/filebeat/processors/decode_azure/fields.go new file mode 100644 index 00000000000..90f04b688d4 --- /dev/null +++ b/x-pack/filebeat/processors/decode_azure/fields.go @@ -0,0 +1,23 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. + +package decode_azure + +import ( + "github.com/elastic/beats/libbeat/asset" +) + +func init() { + if err := asset.SetFields("filebeat", "decode_azure", asset.ModuleFieldsPri, AssetDecodeAzure); err != nil { + panic(err) + } +} + +// AssetDecodeAzure returns asset data. +// This is the base64 encoded gzipped contents of processors/decode_azure. +func AssetDecodeAzure() string { + return "eJwsy7ENQiEQgOH+pvhLLXQACpsn7EHgTIgP7gVOE53exgG+C0/9BPL3NVXAm+8auGuxqmwxcUwrupZNHk33ugSqrjLb4c1G4CYAm/Vug/jW4SSbPTunLaYzNXu+Cn8dRH4BAAD//1ILJAE=" +} From 1bfadb2b1c1eacdfd20fb5468d148b05481779d1 Mon Sep 17 00:00:00 2001 From: Mariana Date: Tue, 24 Sep 2019 12:35:28 +0200 Subject: [PATCH 06/21] Adding azure filebeat input --- filebeat/docs/fields.asciidoc | 4 +- filebeat/input/kafka/config.go | 10 +- filebeat/input/kafka/input.go | 6 +- x-pack/filebeat/filebeat.reference.yml | 5 + x-pack/filebeat/filebeat.yml | 5 +- x-pack/filebeat/include/list.go | 2 +- x-pack/filebeat/input/azure/input.go | 336 ++++++++++++++++++ x-pack/filebeat/module/azure/_meta/config.yml | 5 + .../activitylogs/config/activitylogs.yml | 13 + .../azure/activitylogs/config/kafka.yml | 13 - .../azure/activitylogs/ingest/pipeline.json | 10 - .../module/azure/activitylogs/manifest.yml | 8 +- .../cisco/shared/gen-ecs-mapping-docs.go | 3 +- .../{azure.yml => azure.yml.disabled} | 5 + .../processors/decode_azure/_meta/fields.yml | 6 - .../processors/decode_azure/decode_azure.go | 201 ----------- .../processors/decode_azure/fields.go | 23 -- 17 files changed, 385 insertions(+), 270 deletions(-) create mode 100644 x-pack/filebeat/input/azure/input.go create mode 100644 x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml delete mode 100644 x-pack/filebeat/module/azure/activitylogs/config/kafka.yml rename x-pack/filebeat/modules.d/{azure.yml => azure.yml.disabled} (67%) delete mode 100644 x-pack/filebeat/processors/decode_azure/_meta/fields.yml delete mode 100644 x-pack/filebeat/processors/decode_azure/decode_azure.go delete mode 100644 x-pack/filebeat/processors/decode_azure/fields.go diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 75cf6da0e8c..aff526a6ec9 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -1263,9 +1263,9 @@ Fields for azure Activity logs. [[exported-fields-azure]] -== Decode CEF processor fields fields +== Decode azure processor fields fields -Common Event Format (CEF) data. +Common Event Format (azure) data. [[exported-fields-beat-common]] diff --git a/filebeat/input/kafka/config.go b/filebeat/input/kafka/config.go index ddc505bf4c7..216216fabdb 100644 --- a/filebeat/input/kafka/config.go +++ b/filebeat/input/kafka/config.go @@ -32,7 +32,7 @@ import ( "github.com/elastic/beats/libbeat/outputs" ) -type kafkaInputConfig struct { +type KafkaInputConfig struct { // Kafka hosts with port, e.g. "localhost:9092" Hosts []string `config:"hosts" validate:"required"` Topics []string `config:"topics" validate:"required"` @@ -103,8 +103,8 @@ var ( // The default config for the kafka input. When in doubt, default values // were chosen to match sarama's defaults. -func defaultConfig() kafkaInputConfig { - return kafkaInputConfig{ +func DefaultConfig() KafkaInputConfig { + return KafkaInputConfig{ Version: kafka.Version("1.0.0"), InitialOffset: initialOffsetOldest, ClientID: "filebeat", @@ -128,7 +128,7 @@ func defaultConfig() kafkaInputConfig { } // Validate validates the config. -func (c *kafkaInputConfig) Validate() error { +func (c *KafkaInputConfig) Validate() error { if len(c.Hosts) == 0 { return errors.New("no hosts configured") } @@ -143,7 +143,7 @@ func (c *kafkaInputConfig) Validate() error { return nil } -func newSaramaConfig(config kafkaInputConfig) (*sarama.Config, error) { +func NewSaramaConfig(config KafkaInputConfig) (*sarama.Config, error) { k := sarama.NewConfig() version, ok := config.Version.Get() diff --git a/filebeat/input/kafka/input.go b/filebeat/input/kafka/input.go index 98b7f15c1f0..48d294764c6 100644 --- a/filebeat/input/kafka/input.go +++ b/filebeat/input/kafka/input.go @@ -46,7 +46,7 @@ func init() { // Input contains the input and its config type kafkaInput struct { - config kafkaInputConfig + config KafkaInputConfig saramaConfig *sarama.Config context input.Context outlet channel.Outleter @@ -62,7 +62,7 @@ func NewInput( inputContext input.Context, ) (input.Input, error) { - config := defaultConfig() + config := DefaultConfig() if err := cfg.Unpack(&config); err != nil { return nil, errors.Wrap(err, "reading kafka input config") } @@ -85,7 +85,7 @@ func NewInput( return nil, err } - saramaConfig, err := newSaramaConfig(config) + saramaConfig, err := NewSaramaConfig(config) if err != nil { return nil, errors.Wrap(err, "initializing Sarama config") } diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index a6cc9c3019e..5e4c05d67de 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -92,6 +92,11 @@ filebeat.modules: # All logs activitylogs: enabled: true + var: + eventhubs_namespace: "" + topics: ["insights-operational-logs"] + consumer_group: "$Default" + connection_string: "" # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. diff --git a/x-pack/filebeat/filebeat.yml b/x-pack/filebeat/filebeat.yml index 16d99954455..d02b4d161fb 100644 --- a/x-pack/filebeat/filebeat.yml +++ b/x-pack/filebeat/filebeat.yml @@ -143,7 +143,7 @@ setup.kibana: #================================ Outputs ===================================== # Configure what output to use when sending the data collected by the beat. -filebeat.overwrite_pipelines: true + #-------------------------- Elasticsearch output ------------------------------ output.elasticsearch: # Array of hosts to connect to. @@ -183,8 +183,7 @@ processors: # Sets log level. The default log level is info. # Available log levels are: error, warning, info, debug -logging.level: debug -setup.ilm.enabled: false +#logging.level: debug # At debug level, you can selectively enable logging only for some components. # To enable all selectors use ["*"]. Examples of other selectors are "beat", diff --git a/x-pack/filebeat/include/list.go b/x-pack/filebeat/include/list.go index ec269a10f06..b672f30181b 100644 --- a/x-pack/filebeat/include/list.go +++ b/x-pack/filebeat/include/list.go @@ -8,6 +8,7 @@ package include import ( // Import packages that need to register themselves. + _ "github.com/elastic/beats/x-pack/filebeat/input/azure" _ "github.com/elastic/beats/x-pack/filebeat/input/googlepubsub" _ "github.com/elastic/beats/x-pack/filebeat/input/netflow" _ "github.com/elastic/beats/x-pack/filebeat/input/s3" @@ -26,6 +27,5 @@ import ( _ "github.com/elastic/beats/x-pack/filebeat/module/rabbitmq" _ "github.com/elastic/beats/x-pack/filebeat/module/suricata" _ "github.com/elastic/beats/x-pack/filebeat/module/zeek" - _ "github.com/elastic/beats/x-pack/filebeat/processors/decode_azure" _ "github.com/elastic/beats/x-pack/filebeat/processors/decode_cef" ) diff --git a/x-pack/filebeat/input/azure/input.go b/x-pack/filebeat/input/azure/input.go new file mode 100644 index 00000000000..528f27763b5 --- /dev/null +++ b/x-pack/filebeat/input/azure/input.go @@ -0,0 +1,336 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package azure + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "sync" + "time" + + kafka2 "github.com/elastic/beats/filebeat/input/kafka" + + "github.com/Shopify/sarama" + + "github.com/elastic/beats/filebeat/channel" + "github.com/elastic/beats/filebeat/input" + "github.com/elastic/beats/libbeat/beat" + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/common/backoff" + "github.com/elastic/beats/libbeat/common/kafka" + "github.com/elastic/beats/libbeat/logp" + + "github.com/pkg/errors" +) + +func init() { + err := input.Register("azure", NewInput) + if err != nil { + panic(err) + } +} + +// Input contains the input and its config +type azureInput struct { + config kafka2.KafkaInputConfig + saramaConfig *sarama.Config + context input.Context + outlet channel.Outleter + saramaWaitGroup sync.WaitGroup // indicates a sarama consumer group is active + log *logp.Logger + runOnce sync.Once +} + +// NewInput creates a new kafka input +func NewInput( + cfg *common.Config, + connector channel.Connector, + inputContext input.Context, +) (input.Input, error) { + + config := kafka2.DefaultConfig() + if err := cfg.Unpack(&config); err != nil { + return nil, errors.Wrap(err, "reading kafka input config") + } + + out, err := connector.ConnectWith(cfg, beat.ClientConfig{ + Processing: beat.ProcessingConfig{ + DynamicFields: inputContext.DynamicFields, + }, + ACKEvents: func(events []interface{}) { + for _, event := range events { + if meta, ok := event.(eventMeta); ok { + meta.handler.ack(meta.message) + } + } + }, + CloseRef: doneChannelContext(inputContext.Done), + WaitClose: config.WaitClose, + }) + if err != nil { + return nil, err + } + + saramaConfig, err := kafka2.NewSaramaConfig(config) + if err != nil { + return nil, errors.Wrap(err, "initializing Sarama config") + } + + input := &azureInput{ + config: config, + saramaConfig: saramaConfig, + context: inputContext, + outlet: out, + log: logp.NewLogger("kafka input").With("hosts", config.Hosts), + } + + return input, nil +} + +func (input *azureInput) runConsumerGroup( + context context.Context, consumerGroup sarama.ConsumerGroup, +) { + handler := &groupHandler{ + version: input.config.Version, + outlet: input.outlet, + } + + input.saramaWaitGroup.Add(1) + defer func() { + consumerGroup.Close() + input.saramaWaitGroup.Done() + }() + + // Listen asynchronously to any errors during the consume process + go func() { + for err := range consumerGroup.Errors() { + input.log.Errorw("Error reading from kafka", "error", err) + } + }() + + err := consumerGroup.Consume(context, input.config.Topics, handler) + if err != nil { + input.log.Errorw("Kafka consume error", "error", err) + } +} + +// Run starts the input by scanning for incoming messages and errors. +func (input *azureInput) Run() { + input.runOnce.Do(func() { + go func() { + // Sarama uses standard go contexts to control cancellation, so we need + // to wrap our input context channel in that interface. + context := doneChannelContext(input.context.Done) + + // If the consumer fails to connect, we use exponential backoff with + // jitter up to 8 * the initial backoff interval. + backoff := backoff.NewEqualJitterBackoff( + input.context.Done, + input.config.ConnectBackoff, + 8*input.config.ConnectBackoff) + + for context.Err() == nil { + // Connect to Kafka with a new consumer group. + consumerGroup, err := sarama.NewConsumerGroup( + input.config.Hosts, input.config.GroupID, input.saramaConfig) + if err != nil { + input.log.Errorw( + "Error initializing kafka consumer group", "error", err) + backoff.Wait() + continue + } + // We've successfully connected, reset the backoff timer. + backoff.Reset() + + // We have a connected consumer group now, try to start the main event + // loop by calling Consume (which starts an asynchronous consumer). + // In an ideal run, this function never returns until shutdown; if it + // does, it means the errors have been logged and the consumer group + // has been closed, so we try creating a new one in the next iteration. + input.runConsumerGroup(context, consumerGroup) + } + }() + }) +} + +// Stop doesn't need to do anything because the kafka consumer group and the +// input's outlet both have a context based on input.context.Done and will +// shut themselves down, since the done channel is already closed as part of +// the shutdown process in Runner.Stop(). +func (input *azureInput) Stop() { +} + +// Wait should shut down the input and wait for it to complete, however (see +// Stop above) we don't need to take actions to shut down as long as the +// input.config.Done channel is closed, so we just make a (currently no-op) +// call to Stop() and then wait for sarama to signal completion. +func (input *azureInput) Wait() { + input.Stop() + // Wait for sarama to shut down + input.saramaWaitGroup.Wait() +} + +func arrayForKafkaHeaders(headers []*sarama.RecordHeader) []string { + array := []string{} + for _, header := range headers { + // Rather than indexing headers in the same object structure Kafka does + // (which would give maximal fidelity, but would be effectively unsearchable + // in elasticsearch and kibana) we compromise by serializing them all as + // strings in the form ": ". For this we need to mask + // occurrences of ":" in the original key, which we expect to be uncommon. + // We may consider another approach in the future when it's more clear what + // the most common use cases are. + key := strings.ReplaceAll(string(header.Key), ":", "_") + value := string(header.Value) + array = append(array, fmt.Sprintf("%s: %s", key, value)) + } + return array +} + +// A barebones implementation of context.Context wrapped around the done +// channels that are more common in the beats codebase. +// TODO(faec): Generalize this to a common utility in a shared library +// (https://github.com/elastic/beats/issues/13125). +type channelCtx <-chan struct{} + +func doneChannelContext(ch <-chan struct{}) context.Context { + return channelCtx(ch) +} + +func (c channelCtx) Deadline() (deadline time.Time, ok bool) { return } +func (c channelCtx) Done() <-chan struct{} { + return (<-chan struct{})(c) +} +func (c channelCtx) Err() error { + select { + case <-c: + return context.Canceled + default: + return nil + } +} +func (c channelCtx) Value(key interface{}) interface{} { return nil } + +// The group handler for the sarama consumer group interface. In addition to +// providing the basic consumption callbacks needed by sarama, groupHandler is +// also currently responsible for marshalling kafka messages into beat.Event, +// and passing ACKs from the output channel back to the kafka cluster. +type groupHandler struct { + sync.Mutex + version kafka.Version + session sarama.ConsumerGroupSession + outlet channel.Outleter +} + +// The metadata attached to incoming events so they can be ACKed once they've +// been successfully sent. +type eventMeta struct { + handler *groupHandler + message *sarama.ConsumerMessage +} + +func (h *groupHandler) createEvent( + sess sarama.ConsumerGroupSession, + claim sarama.ConsumerGroupClaim, + message *sarama.ConsumerMessage, +) []beat.Event { + timestamp := time.Now() + kafkaFields := common.MapStr{ + "topic": claim.Topic(), + "partition": claim.Partition(), + "offset": message.Offset, + "key": string(message.Key), + } + + version, versionOk := h.version.Get() + if versionOk && version.IsAtLeast(sarama.V0_10_0_0) { + timestamp = message.Timestamp + if !message.BlockTimestamp.IsZero() { + kafkaFields["block_timestamp"] = message.BlockTimestamp + } + } + if versionOk && version.IsAtLeast(sarama.V0_11_0_0) { + kafkaFields["headers"] = arrayForKafkaHeaders(message.Headers) + } + list := parseMultipleMessages(message.Value) + var events []beat.Event + for _, messageItem := range list { + event := beat.Event{ + Timestamp: timestamp, + Fields: common.MapStr{ + "message": messageItem, + "azure": kafkaFields, + }, + Private: eventMeta{ + handler: h, + message: message, + }, + } + events = append(events, event) + } + + return events +} + +func parseMultipleMessages(bMessage []byte) []string { + var messages []string + originalMessage := []string{string(bMessage)} + if !json.Valid(bMessage) { + return originalMessage + } + var obj common.MapStr + err := json.Unmarshal(bMessage, &obj) + if err != nil { + return originalMessage + } + if obj["records"] != nil && len(obj["records"].([]interface{})) > 0 { + msgs := obj["records"].([]interface{}) + for _, ms := range msgs { + js, err := json.Marshal(ms) + if err != nil { + return originalMessage + } + messages = append(messages, string(js)) + } + return messages + } + return originalMessage +} + +func (h *groupHandler) Setup(session sarama.ConsumerGroupSession) error { + h.Lock() + h.session = session + h.Unlock() + return nil +} + +func (h *groupHandler) Cleanup(_ sarama.ConsumerGroupSession) error { + h.Lock() + h.session = nil + h.Unlock() + return nil +} + +// ack informs the kafka cluster that this message has been consumed. Called +// from the input's ACKEvents handler. +func (h *groupHandler) ack(message *sarama.ConsumerMessage) { + h.Lock() + defer h.Unlock() + if h.session != nil { + h.session.MarkMessage(message, "") + } +} + +func (h *groupHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { + for msg := range claim.Messages() { + events := h.createEvent(sess, claim, msg) + for _, event := range events { + h.outlet.OnEvent(event) + } + } + return nil +} diff --git a/x-pack/filebeat/module/azure/_meta/config.yml b/x-pack/filebeat/module/azure/_meta/config.yml index 90e310e2e43..9d77c6fbb4c 100644 --- a/x-pack/filebeat/module/azure/_meta/config.yml +++ b/x-pack/filebeat/module/azure/_meta/config.yml @@ -2,6 +2,11 @@ # All logs activitylogs: enabled: true + var: + eventhubs_namespace: "" + topics: ["insights-operational-logs"] + consumer_group: "$Default" + connection_string: "" # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. diff --git a/x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml b/x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml new file mode 100644 index 00000000000..8fa8ca064e2 --- /dev/null +++ b/x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml @@ -0,0 +1,13 @@ +type: azure +hosts: {{ .eventhubs_namespace }} +topics: {{ .topics }} +group_id: {{ .consumer_group }} + +username: "$ConnectionString" +password: {{ .connection_string }} +ssl.enabled: true + +processors: +- decode_json_fields: + fields: ["message"] + target: "activitylogs" diff --git a/x-pack/filebeat/module/azure/activitylogs/config/kafka.yml b/x-pack/filebeat/module/azure/activitylogs/config/kafka.yml deleted file mode 100644 index b3d1e363423..00000000000 --- a/x-pack/filebeat/module/azure/activitylogs/config/kafka.yml +++ /dev/null @@ -1,13 +0,0 @@ -type: kafka -hosts: ["{}.servicebus.windows.net:9093"] -topics: ["insights-operational-logs"] -group_id: "$Default" - -username: "$ConnectionString" -password: "" -ssl.enabled: true - -processors: -- decode_json_fields: - fields: ["message"] - target: "" diff --git a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json index 97a86859079..770f447d349 100644 --- a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json +++ b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json @@ -1,16 +1,6 @@ { "description": "Pipeline for parsing azure activity logs.", "processors": [ - { - "foreach": { - "field": "records", - "processor" : { - "uppercase" : { - "field" : "_ingest._value.category" - } - } - } - }, { "remove": { "field":"message", diff --git a/x-pack/filebeat/module/azure/activitylogs/manifest.yml b/x-pack/filebeat/module/azure/activitylogs/manifest.yml index 65d744199b1..e85d461325e 100644 --- a/x-pack/filebeat/module/azure/activitylogs/manifest.yml +++ b/x-pack/filebeat/module/azure/activitylogs/manifest.yml @@ -2,7 +2,11 @@ module_version: 1.0 var: - name: input - default: kafka + default: azure + - name: topics + default: "insights-operational-logs" + - name: consumer_group + default: "$Default" ingest_pipeline: ingest/pipeline.json -input: config/kafka.yml +input: config/activitylogs.yml diff --git a/x-pack/filebeat/module/cisco/shared/gen-ecs-mapping-docs.go b/x-pack/filebeat/module/cisco/shared/gen-ecs-mapping-docs.go index 7e5ed04bd8f..a6d54e5a8bf 100644 --- a/x-pack/filebeat/module/cisco/shared/gen-ecs-mapping-docs.go +++ b/x-pack/filebeat/module/cisco/shared/gen-ecs-mapping-docs.go @@ -10,11 +10,12 @@ import ( "encoding/csv" "flag" "fmt" - "github.com/elastic/beats/libbeat/common" "io" "os" "sort" + "github.com/elastic/beats/libbeat/common" + "github.com/pkg/errors" ) diff --git a/x-pack/filebeat/modules.d/azure.yml b/x-pack/filebeat/modules.d/azure.yml.disabled similarity index 67% rename from x-pack/filebeat/modules.d/azure.yml rename to x-pack/filebeat/modules.d/azure.yml.disabled index 5f9c1de9bb4..2a0154535b5 100644 --- a/x-pack/filebeat/modules.d/azure.yml +++ b/x-pack/filebeat/modules.d/azure.yml.disabled @@ -5,6 +5,11 @@ # All logs activitylogs: enabled: true + var: + eventhubs_namespace: "" + topics: ["insights-operational-logs"] + consumer_group: "$Default" + connection_string: "" # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. diff --git a/x-pack/filebeat/processors/decode_azure/_meta/fields.yml b/x-pack/filebeat/processors/decode_azure/_meta/fields.yml deleted file mode 100644 index a979dcdb5f7..00000000000 --- a/x-pack/filebeat/processors/decode_azure/_meta/fields.yml +++ /dev/null @@ -1,6 +0,0 @@ -- key: azure - title: Decode azure processor fields - description: > - Common Event Format (azure) data. - fields: - diff --git a/x-pack/filebeat/processors/decode_azure/decode_azure.go b/x-pack/filebeat/processors/decode_azure/decode_azure.go deleted file mode 100644 index c18c26a8cb8..00000000000 --- a/x-pack/filebeat/processors/decode_azure/decode_azure.go +++ /dev/null @@ -1,201 +0,0 @@ -package decode_azure - -import ( - "encoding/json" - "github.com/elastic/beats/libbeat/beat" - "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/libbeat/common/cfgwarn" - "github.com/elastic/beats/libbeat/logp" - "github.com/elastic/beats/libbeat/processors" - "github.com/elastic/beats/x-pack/filebeat/processors/decode_cef/cef" - "github.com/pkg/errors" - "go.uber.org/multierr" - "strconv" - "strings" -) - -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - - - -const ( - procName = "decode_azure" - logName = "processor." + procName -) - -type config struct { - Field string `config:"field"` // Source field containing the CEF message. - TargetField string `config:"target_field"` // Target field for the CEF object. - IgnoreMissing bool `config:"ignore_missing"` // Ignore missing source field. - IgnoreFailure bool `config:"ignore_failure"` // Ignore failures when the source field does not contain a CEF message. Parse errors do not cause failures, but are added to error.message. - ID string `config:"id"` // Instance ID for debugging purposes. - ECS bool `config:"ecs"` // Generate ECS fields. -} - -func defaultConfig() config { - return config{ - Field: "message", - TargetField: "cef", - ECS: true, - } -} - -func init() { - processors.RegisterPlugin(procName, New) -} - -type processor struct { - config - log *logp.Logger -} - -// New constructs a new processor built from ucfg config. -func New(cfg *common.Config) (processors.Processor, error) { - c := defaultConfig() - if err := cfg.Unpack(&c); err != nil { - return nil, errors.Wrap(err, "fail to unpack the "+procName+" processor configuration") - } - - return newDecodeCEF(c) -} - -func newDecodeCEF(c config) (*processor, error) { - cfgwarn.Beta("The " + procName + " processor is a beta feature.") - - log := logp.NewLogger(logName) - if c.ID != "" { - log = log.With("instance_id", c.ID) - } - - return &processor{config: c, log: log}, nil -} - -func (p *processor) String() string { - json, _ := json.Marshal(p.config) - return procName + "=" + string(json) -} - -func (p *processor) Run(event *beat.Event) (*beat.Event, error) { - v, err := event.GetValue(p.Field) - if err != nil { - if p.IgnoreMissing { - return event, nil - } - return event, errors.Wrapf(err, "decode_cef field [%v] not found", p.Field) - } - - cefData, ok := v.(string) - if !ok { - if p.IgnoreFailure { - return event, nil - } - return event, errors.Wrapf(err, "decode_cef field [%v] is not a string", p.Field) - } - - // Ignore any leading data before the CEF header. - idx := strings.Index(cefData, "CEF:") - if idx == -1 { - if p.IgnoreFailure { - return event, nil - } - return event, errors.Errorf("decode_cef field [%v] does not contain a CEF header", p.Field) - } - cefData = cefData[idx:] - - // If the version < 0 after parsing then none of the data is valid so return here. - var ce cef.Event - if err = ce.Unpack([]byte(cefData), cef.WithFullExtensionNames()); ce.Version < 0 && err != nil { - if p.IgnoreFailure { - return event, nil - } - return event, errors.Wrap(err, "decode_cef failed to parse message") - } - - cefErrors := multierr.Errors(err) - cefObject := toCEFObject(&ce) - event.PutValue(p.TargetField, cefObject) - - - // Add all parsing/conversion errors to error.message. - for _, cefError := range cefErrors { - if err := appendErrorMessage(event.Fields, cefError.Error()); err != nil { - p.log.Warn("Failed adding CEF errors to event.", "error", err) - break - } - } - - return event, nil -} - -func toCEFObject(cefEvent *cef.Event) common.MapStr { - // Add CEF header fields. - cefObject := common.MapStr{"version": strconv.Itoa(cefEvent.Version)} - if cefEvent.DeviceVendor != "" { - cefObject.Put("device.vendor", cefEvent.DeviceVendor) - } - if cefEvent.DeviceProduct != "" { - cefObject.Put("device.product", cefEvent.DeviceProduct) - } - if cefEvent.DeviceVersion != "" { - cefObject.Put("device.version", cefEvent.DeviceVersion) - } - if cefEvent.DeviceEventClassID != "" { - cefObject.Put("device.event_class_id", cefEvent.DeviceEventClassID) - } - if cefEvent.Name != "" { - cefObject.Put("name", cefEvent.Name) - } - if cefEvent.Severity != "" { - cefObject.Put("severity", cefEvent.Severity) - } - - // Add CEF extensions (key-value pairs). - if len(cefEvent.Extensions) > 0 { - extensions := make(common.MapStr, len(cefEvent.Extensions)) - cefObject.Put("extensions", extensions) - for k, v := range cefEvent.Extensions { - extensions.Put(k, v) - } - } - - return cefObject -} - - - -func appendErrorMessage(m common.MapStr, msg string) error { - const field = "error.message" - list, _ := m.GetValue(field) - - switch v := list.(type) { - case nil: - m.Put(field, msg) - case string: - if msg != v { - m.Put(field, []string{v, msg}) - } - case []string: - for _, existingTag := range v { - if msg == existingTag { - // Duplicate - return nil - } - } - m.Put(field, append(v, msg)) - case []interface{}: - for _, existingTag := range v { - if msg == existingTag { - // Duplicate - return nil - } - } - m.Put(field, append(v, msg)) - default: - return errors.Errorf("unexpected type %T found for %v field", list, field) - } - return nil -} - - diff --git a/x-pack/filebeat/processors/decode_azure/fields.go b/x-pack/filebeat/processors/decode_azure/fields.go deleted file mode 100644 index 90f04b688d4..00000000000 --- a/x-pack/filebeat/processors/decode_azure/fields.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. - -package decode_azure - -import ( - "github.com/elastic/beats/libbeat/asset" -) - -func init() { - if err := asset.SetFields("filebeat", "decode_azure", asset.ModuleFieldsPri, AssetDecodeAzure); err != nil { - panic(err) - } -} - -// AssetDecodeAzure returns asset data. -// This is the base64 encoded gzipped contents of processors/decode_azure. -func AssetDecodeAzure() string { - return "eJwsy7ENQiEQgOH+pvhLLXQACpsn7EHgTIgP7gVOE53exgG+C0/9BPL3NVXAm+8auGuxqmwxcUwrupZNHk33ugSqrjLb4c1G4CYAm/Vug/jW4SSbPTunLaYzNXu+Cn8dRH4BAAD//1ILJAE=" -} From 14feac4804c2909c38a68e93746c90288f1b2c85 Mon Sep 17 00:00:00 2001 From: Mariana Date: Thu, 26 Sep 2019 15:36:18 +0200 Subject: [PATCH 07/21] Modify kafka input --- filebeat/input/kafka/azure_log_patterns.go | 62 ++++ filebeat/input/kafka/config.go | 43 +-- filebeat/input/kafka/input.go | 72 +++- x-pack/filebeat/input/azure/config.go | 14 + x-pack/filebeat/input/azure/input.go | 313 +----------------- .../azure/activitylogs/_meta/fields.yml | 96 ++++++ .../activitylogs/config/activitylogs.yml | 3 +- .../azure/activitylogs/ingest/pipeline.json | 27 +- .../azure/activitylogs/test/activityLog.json | 0 .../test/activity_log_expected.json | 0 x-pack/filebeat/module/azure/fields.go | 2 +- 11 files changed, 279 insertions(+), 353 deletions(-) create mode 100644 filebeat/input/kafka/azure_log_patterns.go create mode 100644 x-pack/filebeat/input/azure/config.go create mode 100644 x-pack/filebeat/module/azure/activitylogs/test/activityLog.json create mode 100644 x-pack/filebeat/module/azure/activitylogs/test/activity_log_expected.json diff --git a/filebeat/input/kafka/azure_log_patterns.go b/filebeat/input/kafka/azure_log_patterns.go new file mode 100644 index 00000000000..1b2b45b5595 --- /dev/null +++ b/filebeat/input/kafka/azure_log_patterns.go @@ -0,0 +1,62 @@ +package kafka + +import "time" + +const ( + ActivityLogs = "ActivityLogs" + AuditLogs = "AuditLogs" +) + +// ActivityLogs structure matches the azure activity log format +type ActivityLog struct { + Time time.Time `json:"time"` + ResourceID string `json:"resourceId"` + OperationName string `json:"operationName"` + Category string `json:"category"` + ResultType string `json:"resultType"` + ResultSignature string `json:"resultSignature"` + DurationMs int `json:"durationMs"` + CallerIPAddress string `json:"callerIpAddress"` + CorrelationID string `json:"correlationId"` + Identity struct { + Authorization struct { + Scope string `json:"scope"` + Action string `json:"action"` + Evidence struct { + Role string `json:"role"` + RoleAssignmentScope string `json:"roleAssignmentScope"` + RoleAssignmentID string `json:"roleAssignmentId"` + RoleDefinitionID string `json:"roleDefinitionId"` + PrincipalID string `json:"principalId"` + PrincipalType string `json:"principalType"` + } `json:"evidence"` + } `json:"authorization"` + Claims struct { + Aud string `json:"aud"` + Iss string `json:"iss"` + Iat string `json:"iat"` + Nbf string `json:"nbf"` + Exp string `json:"exp"` + Aio string `json:"aio"` + Appid string `json:"appid"` + Appidacr string `json:"appidacr"` + HTTPSchemasMicrosoftComIdentityClaimsIdentityprovider string `json:"http://schemas.microsoft.com/identity/claims/identityprovider"` + HTTPSchemasMicrosoftComIdentityClaimsObjectidentifier string `json:"http://schemas.microsoft.com/identity/claims/objectidentifier"` + HTTPSchemasXmlsoapOrgWs200505IdentityClaimsNameidentifier string `json:"http://schemas.xmlsoap.org/ws/2005/05/identity/claims/nameidentifier"` + HTTPSchemasMicrosoftComIdentityClaimsTenantid string `json:"http://schemas.microsoft.com/identity/claims/tenantid"` + Uti string `json:"uti"` + Ver string `json:"ver"` + } `json:"claims"` + } `json:"identity"` + Level string `json:"level"` + Location string `json:"location"` + Properties struct { + StatusCode string `json:"statusCode"` + ServiceRequestID interface{} `json:"serviceRequestId"` + StatusMessage string `json:"statusMessage"` + } `json:"properties,omitempty"` +} + +type AzureActivityLogs struct { + Records []ActivityLog `json:"records"` +} diff --git a/filebeat/input/kafka/config.go b/filebeat/input/kafka/config.go index 216216fabdb..4bf137ef69c 100644 --- a/filebeat/input/kafka/config.go +++ b/filebeat/input/kafka/config.go @@ -32,24 +32,25 @@ import ( "github.com/elastic/beats/libbeat/outputs" ) -type KafkaInputConfig struct { +type kafkaInputConfig struct { // Kafka hosts with port, e.g. "localhost:9092" - Hosts []string `config:"hosts" validate:"required"` - Topics []string `config:"topics" validate:"required"` - GroupID string `config:"group_id" validate:"required"` - ClientID string `config:"client_id"` - Version kafka.Version `config:"version"` - InitialOffset initialOffset `config:"initial_offset"` - ConnectBackoff time.Duration `config:"connect_backoff" validate:"min=0"` - ConsumeBackoff time.Duration `config:"consume_backoff" validate:"min=0"` - WaitClose time.Duration `config:"wait_close" validate:"min=0"` - MaxWaitTime time.Duration `config:"max_wait_time"` - IsolationLevel isolationLevel `config:"isolation_level"` - Fetch kafkaFetch `config:"fetch"` - Rebalance kafkaRebalance `config:"rebalance"` - TLS *tlscommon.Config `config:"ssl"` - Username string `config:"username"` - Password string `config:"password"` + Hosts []string `config:"hosts" validate:"required"` + Topics []string `config:"topics" validate:"required"` + GroupID string `config:"group_id" validate:"required"` + ClientID string `config:"client_id"` + Version kafka.Version `config:"version"` + InitialOffset initialOffset `config:"initial_offset"` + ConnectBackoff time.Duration `config:"connect_backoff" validate:"min=0"` + ConsumeBackoff time.Duration `config:"consume_backoff" validate:"min=0"` + WaitClose time.Duration `config:"wait_close" validate:"min=0"` + MaxWaitTime time.Duration `config:"max_wait_time"` + IsolationLevel isolationLevel `config:"isolation_level"` + Fetch kafkaFetch `config:"fetch"` + Rebalance kafkaRebalance `config:"rebalance"` + TLS *tlscommon.Config `config:"ssl"` + Username string `config:"username"` + Password string `config:"password"` + AzureLogs string `config:"azure_logs"` } type kafkaFetch struct { @@ -103,8 +104,8 @@ var ( // The default config for the kafka input. When in doubt, default values // were chosen to match sarama's defaults. -func DefaultConfig() KafkaInputConfig { - return KafkaInputConfig{ +func DefaultConfig() kafkaInputConfig { + return kafkaInputConfig{ Version: kafka.Version("1.0.0"), InitialOffset: initialOffsetOldest, ClientID: "filebeat", @@ -128,7 +129,7 @@ func DefaultConfig() KafkaInputConfig { } // Validate validates the config. -func (c *KafkaInputConfig) Validate() error { +func (c *kafkaInputConfig) Validate() error { if len(c.Hosts) == 0 { return errors.New("no hosts configured") } @@ -143,7 +144,7 @@ func (c *KafkaInputConfig) Validate() error { return nil } -func NewSaramaConfig(config KafkaInputConfig) (*sarama.Config, error) { +func NewSaramaConfig(config kafkaInputConfig) (*sarama.Config, error) { k := sarama.NewConfig() version, ok := config.Version.Get() diff --git a/filebeat/input/kafka/input.go b/filebeat/input/kafka/input.go index 48d294764c6..145965bcce4 100644 --- a/filebeat/input/kafka/input.go +++ b/filebeat/input/kafka/input.go @@ -19,6 +19,7 @@ package kafka import ( "context" + "encoding/json" "fmt" "strings" "sync" @@ -46,7 +47,7 @@ func init() { // Input contains the input and its config type kafkaInput struct { - config KafkaInputConfig + config kafkaInputConfig saramaConfig *sarama.Config context input.Context outlet channel.Outleter @@ -104,9 +105,11 @@ func NewInput( func (input *kafkaInput) runConsumerGroup( context context.Context, consumerGroup sarama.ConsumerGroup, ) { + handler := &groupHandler{ version: input.config.Version, outlet: input.outlet, + logType: input.config.AzureLogs, } input.saramaWaitGroup.Add(1) @@ -234,6 +237,7 @@ type groupHandler struct { version kafka.Version session sarama.ConsumerGroupSession outlet channel.Outleter + logType string } // The metadata attached to incoming events so they can be ACKed once they've @@ -247,7 +251,7 @@ func (h *groupHandler) createEvent( sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim, message *sarama.ConsumerMessage, -) beat.Event { +) []beat.Event { timestamp := time.Now() kafkaFields := common.MapStr{ "topic": claim.Topic(), @@ -266,19 +270,27 @@ func (h *groupHandler) createEvent( if versionOk && version.IsAtLeast(sarama.V0_11_0_0) { kafkaFields["headers"] = arrayForKafkaHeaders(message.Headers) } - event := beat.Event{ - Timestamp: timestamp, - Fields: common.MapStr{ - "message": string(message.Value), - "kafka": kafkaFields, - }, - Private: eventMeta{ - handler: h, - message: message, - }, - } - return event + // if azure input, then a check for the message is done regarding the list of events + + var events []beat.Event + messages := h.parseMultipleMessages(message.Value) + for _, msg := range messages { + event := beat.Event{ + Timestamp: timestamp, + Fields: common.MapStr{ + "message": msg, + "kafka": kafkaFields, + }, + Private: eventMeta{ + handler: h, + message: message, + }, + } + events = append(events, event) + + } + return events } func (h *groupHandler) Setup(session sarama.ConsumerGroupSession) error { @@ -307,8 +319,36 @@ func (h *groupHandler) ack(message *sarama.ConsumerMessage) { func (h *groupHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { for msg := range claim.Messages() { - event := h.createEvent(sess, claim, msg) - h.outlet.OnEvent(event) + events := h.createEvent(sess, claim, msg) + for _, event := range events { + h.outlet.OnEvent(event) + } + } + return nil +} + +func (h *groupHandler) parseMultipleMessages(bMessage []byte) []string { + var messages []string + // check if logType has been set + switch h.logType { + // if no log type has been set then the original message should be returned + case "": + default: + return []string{string(bMessage)} + case ActivityLogs: + // if the fileset is activity logs a filtering of the messages should be done as the eventhub can return different types of messages + var obj AzureActivityLogs + err := json.Unmarshal(bMessage, &obj) + if err != nil { + return nil + } + for _, ms := range obj.Records { + js, err := json.Marshal(ms) + if err == nil { + messages = append(messages, string(js)) + } + } + return messages } return nil } diff --git a/x-pack/filebeat/input/azure/config.go b/x-pack/filebeat/input/azure/config.go new file mode 100644 index 00000000000..1a9a3332d1c --- /dev/null +++ b/x-pack/filebeat/input/azure/config.go @@ -0,0 +1,14 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package azure + +var defaultConfig = config{ + YieldEventsFromField: "records", +} + +type config struct { + // YieldEventsFromField (i) + YieldEventsFromField string `config:"yield_events_from_field"` +} diff --git a/x-pack/filebeat/input/azure/input.go b/x-pack/filebeat/input/azure/input.go index 528f27763b5..cb6a5cbed30 100644 --- a/x-pack/filebeat/input/azure/input.go +++ b/x-pack/filebeat/input/azure/input.go @@ -5,26 +5,11 @@ package azure import ( - "context" - "encoding/json" - "fmt" - "strings" - "sync" - "time" - - kafka2 "github.com/elastic/beats/filebeat/input/kafka" - - "github.com/Shopify/sarama" + "github.com/elastic/beats/filebeat/input/kafka" "github.com/elastic/beats/filebeat/channel" "github.com/elastic/beats/filebeat/input" - "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/libbeat/common/backoff" - "github.com/elastic/beats/libbeat/common/kafka" - "github.com/elastic/beats/libbeat/logp" - - "github.com/pkg/errors" ) func init() { @@ -34,303 +19,13 @@ func init() { } } -// Input contains the input and its config -type azureInput struct { - config kafka2.KafkaInputConfig - saramaConfig *sarama.Config - context input.Context - outlet channel.Outleter - saramaWaitGroup sync.WaitGroup // indicates a sarama consumer group is active - log *logp.Logger - runOnce sync.Once -} - // NewInput creates a new kafka input func NewInput( cfg *common.Config, connector channel.Connector, - inputContext input.Context, + context input.Context, ) (input.Input, error) { + // Wrap log input with custom docker settings - config := kafka2.DefaultConfig() - if err := cfg.Unpack(&config); err != nil { - return nil, errors.Wrap(err, "reading kafka input config") - } - - out, err := connector.ConnectWith(cfg, beat.ClientConfig{ - Processing: beat.ProcessingConfig{ - DynamicFields: inputContext.DynamicFields, - }, - ACKEvents: func(events []interface{}) { - for _, event := range events { - if meta, ok := event.(eventMeta); ok { - meta.handler.ack(meta.message) - } - } - }, - CloseRef: doneChannelContext(inputContext.Done), - WaitClose: config.WaitClose, - }) - if err != nil { - return nil, err - } - - saramaConfig, err := kafka2.NewSaramaConfig(config) - if err != nil { - return nil, errors.Wrap(err, "initializing Sarama config") - } - - input := &azureInput{ - config: config, - saramaConfig: saramaConfig, - context: inputContext, - outlet: out, - log: logp.NewLogger("kafka input").With("hosts", config.Hosts), - } - - return input, nil -} - -func (input *azureInput) runConsumerGroup( - context context.Context, consumerGroup sarama.ConsumerGroup, -) { - handler := &groupHandler{ - version: input.config.Version, - outlet: input.outlet, - } - - input.saramaWaitGroup.Add(1) - defer func() { - consumerGroup.Close() - input.saramaWaitGroup.Done() - }() - - // Listen asynchronously to any errors during the consume process - go func() { - for err := range consumerGroup.Errors() { - input.log.Errorw("Error reading from kafka", "error", err) - } - }() - - err := consumerGroup.Consume(context, input.config.Topics, handler) - if err != nil { - input.log.Errorw("Kafka consume error", "error", err) - } -} - -// Run starts the input by scanning for incoming messages and errors. -func (input *azureInput) Run() { - input.runOnce.Do(func() { - go func() { - // Sarama uses standard go contexts to control cancellation, so we need - // to wrap our input context channel in that interface. - context := doneChannelContext(input.context.Done) - - // If the consumer fails to connect, we use exponential backoff with - // jitter up to 8 * the initial backoff interval. - backoff := backoff.NewEqualJitterBackoff( - input.context.Done, - input.config.ConnectBackoff, - 8*input.config.ConnectBackoff) - - for context.Err() == nil { - // Connect to Kafka with a new consumer group. - consumerGroup, err := sarama.NewConsumerGroup( - input.config.Hosts, input.config.GroupID, input.saramaConfig) - if err != nil { - input.log.Errorw( - "Error initializing kafka consumer group", "error", err) - backoff.Wait() - continue - } - // We've successfully connected, reset the backoff timer. - backoff.Reset() - - // We have a connected consumer group now, try to start the main event - // loop by calling Consume (which starts an asynchronous consumer). - // In an ideal run, this function never returns until shutdown; if it - // does, it means the errors have been logged and the consumer group - // has been closed, so we try creating a new one in the next iteration. - input.runConsumerGroup(context, consumerGroup) - } - }() - }) -} - -// Stop doesn't need to do anything because the kafka consumer group and the -// input's outlet both have a context based on input.context.Done and will -// shut themselves down, since the done channel is already closed as part of -// the shutdown process in Runner.Stop(). -func (input *azureInput) Stop() { -} - -// Wait should shut down the input and wait for it to complete, however (see -// Stop above) we don't need to take actions to shut down as long as the -// input.config.Done channel is closed, so we just make a (currently no-op) -// call to Stop() and then wait for sarama to signal completion. -func (input *azureInput) Wait() { - input.Stop() - // Wait for sarama to shut down - input.saramaWaitGroup.Wait() -} - -func arrayForKafkaHeaders(headers []*sarama.RecordHeader) []string { - array := []string{} - for _, header := range headers { - // Rather than indexing headers in the same object structure Kafka does - // (which would give maximal fidelity, but would be effectively unsearchable - // in elasticsearch and kibana) we compromise by serializing them all as - // strings in the form ": ". For this we need to mask - // occurrences of ":" in the original key, which we expect to be uncommon. - // We may consider another approach in the future when it's more clear what - // the most common use cases are. - key := strings.ReplaceAll(string(header.Key), ":", "_") - value := string(header.Value) - array = append(array, fmt.Sprintf("%s: %s", key, value)) - } - return array -} - -// A barebones implementation of context.Context wrapped around the done -// channels that are more common in the beats codebase. -// TODO(faec): Generalize this to a common utility in a shared library -// (https://github.com/elastic/beats/issues/13125). -type channelCtx <-chan struct{} - -func doneChannelContext(ch <-chan struct{}) context.Context { - return channelCtx(ch) -} - -func (c channelCtx) Deadline() (deadline time.Time, ok bool) { return } -func (c channelCtx) Done() <-chan struct{} { - return (<-chan struct{})(c) -} -func (c channelCtx) Err() error { - select { - case <-c: - return context.Canceled - default: - return nil - } -} -func (c channelCtx) Value(key interface{}) interface{} { return nil } - -// The group handler for the sarama consumer group interface. In addition to -// providing the basic consumption callbacks needed by sarama, groupHandler is -// also currently responsible for marshalling kafka messages into beat.Event, -// and passing ACKs from the output channel back to the kafka cluster. -type groupHandler struct { - sync.Mutex - version kafka.Version - session sarama.ConsumerGroupSession - outlet channel.Outleter -} - -// The metadata attached to incoming events so they can be ACKed once they've -// been successfully sent. -type eventMeta struct { - handler *groupHandler - message *sarama.ConsumerMessage -} - -func (h *groupHandler) createEvent( - sess sarama.ConsumerGroupSession, - claim sarama.ConsumerGroupClaim, - message *sarama.ConsumerMessage, -) []beat.Event { - timestamp := time.Now() - kafkaFields := common.MapStr{ - "topic": claim.Topic(), - "partition": claim.Partition(), - "offset": message.Offset, - "key": string(message.Key), - } - - version, versionOk := h.version.Get() - if versionOk && version.IsAtLeast(sarama.V0_10_0_0) { - timestamp = message.Timestamp - if !message.BlockTimestamp.IsZero() { - kafkaFields["block_timestamp"] = message.BlockTimestamp - } - } - if versionOk && version.IsAtLeast(sarama.V0_11_0_0) { - kafkaFields["headers"] = arrayForKafkaHeaders(message.Headers) - } - list := parseMultipleMessages(message.Value) - var events []beat.Event - for _, messageItem := range list { - event := beat.Event{ - Timestamp: timestamp, - Fields: common.MapStr{ - "message": messageItem, - "azure": kafkaFields, - }, - Private: eventMeta{ - handler: h, - message: message, - }, - } - events = append(events, event) - } - - return events -} - -func parseMultipleMessages(bMessage []byte) []string { - var messages []string - originalMessage := []string{string(bMessage)} - if !json.Valid(bMessage) { - return originalMessage - } - var obj common.MapStr - err := json.Unmarshal(bMessage, &obj) - if err != nil { - return originalMessage - } - if obj["records"] != nil && len(obj["records"].([]interface{})) > 0 { - msgs := obj["records"].([]interface{}) - for _, ms := range msgs { - js, err := json.Marshal(ms) - if err != nil { - return originalMessage - } - messages = append(messages, string(js)) - } - return messages - } - return originalMessage -} - -func (h *groupHandler) Setup(session sarama.ConsumerGroupSession) error { - h.Lock() - h.session = session - h.Unlock() - return nil -} - -func (h *groupHandler) Cleanup(_ sarama.ConsumerGroupSession) error { - h.Lock() - h.session = nil - h.Unlock() - return nil -} - -// ack informs the kafka cluster that this message has been consumed. Called -// from the input's ACKEvents handler. -func (h *groupHandler) ack(message *sarama.ConsumerMessage) { - h.Lock() - defer h.Unlock() - if h.session != nil { - h.session.MarkMessage(message, "") - } -} - -func (h *groupHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { - for msg := range claim.Messages() { - events := h.createEvent(sess, claim, msg) - for _, event := range events { - h.outlet.OnEvent(event) - } - } - return nil + return kafka.NewInput(cfg, connector, context) } diff --git a/x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml b/x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml index 859a914d5a9..d4c120cdb98 100644 --- a/x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml +++ b/x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml @@ -3,3 +3,99 @@ description: > Fields for azure Activity logs. fields: + - name: identity + type: group + description: > + The canonical user ID of the owner of the source bucket. + fields: + - name: claims.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Claims + - name: authorization + type: group + description: > + Node allocatable pods + fields: + - name: evidence + type: group + description: > + Node allocatable pods + fields: + - name: roleAssignmentScope + type: keyword + description: > + Role assignment scope + - name: roleDefinitionId + type: keyword + description: > + Role definition ID + - name: role + type: keyword + description: > + Role + - name: roleAssignmentId + type: keyword + description: > + Role assignment ID + - name: principalId + type: keyword + description: > + Principal ID + - name: principalType + type: keyword + description: > + Principal type + - name: scope + type: keyword + description: > + Scope + - name: action + type: keyword + description: > + Action + - name: correlationId + type: keyword + description: > + Correlation ID + - name: resultType + type: keyword + description: > + Result Type + - name: callerIpAddress + type: keyword + description: > + Caller Ip address + - name: resourceID + type: keyword + description: > + Resource ID + - name: level + type: keyword + description: > + Level + - name: operationName + type: keyword + description: > + Operation name + - name: resultSignature + type: keyword + description: > + Result signature + - name: properties.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Properties + - name: location + type: keyword + description: > + Location + - name: category + type: keyword + description: > + Category + diff --git a/x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml b/x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml index 8fa8ca064e2..ebe34ec771c 100644 --- a/x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml +++ b/x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml @@ -2,6 +2,7 @@ type: azure hosts: {{ .eventhubs_namespace }} topics: {{ .topics }} group_id: {{ .consumer_group }} +azure_logs: "ActivityLogs" username: "$ConnectionString" password: {{ .connection_string }} @@ -10,4 +11,4 @@ ssl.enabled: true processors: - decode_json_fields: fields: ["message"] - target: "activitylogs" + target: "" diff --git a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json index 770f447d349..b5c2241a5e4 100644 --- a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json +++ b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json @@ -1,12 +1,29 @@ { "description": "Pipeline for parsing azure activity logs.", "processors": [ - { - "remove": { - "field":"message", - "ignore_missing": true + { + "date": { + "field": "time", + "target_field": "@timestamp", + "ignore_failure": false, + "formats": [ + "ISO8601" + ] + } + }, + { + "rename": { + "field": "resourceId", + "target_field": "resourceID", + "ignore_missing": true + } + }, + { + "remove": { + "field": ["message", "time"], + "ignore_missing": true + } } - } ], "on_failure": [ { diff --git a/x-pack/filebeat/module/azure/activitylogs/test/activityLog.json b/x-pack/filebeat/module/azure/activitylogs/test/activityLog.json new file mode 100644 index 00000000000..e69de29bb2d diff --git a/x-pack/filebeat/module/azure/activitylogs/test/activity_log_expected.json b/x-pack/filebeat/module/azure/activitylogs/test/activity_log_expected.json new file mode 100644 index 00000000000..e69de29bb2d diff --git a/x-pack/filebeat/module/azure/fields.go b/x-pack/filebeat/module/azure/fields.go index 65c676abc24..c7539a97f28 100644 --- a/x-pack/filebeat/module/azure/fields.go +++ b/x-pack/filebeat/module/azure/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAzure returns asset data. // This is the base64 encoded gzipped contents of module/azure. func AssetAzure() string { - return "eJx0z00KwyAQBeC9p3hknx7ARaGb7nqIECdhqI2iY8GevmjsTyCZnb7nh9PjTlljeKVAChAWSxpdPXcKMBTHwF7YLRpnBWDt4uZMsuXJxGRN1DXqsQwP+nFlJHvSmINLvt3smFtmQ43CT5Zs3Ry/4Z56KK9zrT4mF9oClwajyKe/7ucn7wAAAP//cWNQHw==" + return "eJzEl81u4jAQx+88xYhjpfIAHFZCRSsh7XartvfK2EPwYjyW7VClT79yIB8Q54MNy/oWPPn/f2OPneERdpjNgX2lFicAXnqFc5jmz9MJgEDHrTRekp7DtwkAHGPhJ4lUhVc2EpVw83zqETTbYyUXhs8MziGxlJrTLxHNc5kzKe7lQfpMUeLKyZhqq/JxfM/1YUP2lMDiJAxBeVaLvSSp00iB2kufnU220fQQhfG+ReBMk5acKUgdWlgtgTbgtwj0qdEWD45SyxHWKd+hn10IxZDr2FwxuXezh0ZAgU7r38h9ZPo48XGM2mH2SVZ0h33smTFSJ6d3pg/TSHzPsoTxlDO3psRSvyUrv1jQaM0rtiUD/Z9JIDCliDPP1grBkGjitK99HRYPoXA4RoP6WAfyXsPcx11nt6Rw4ZxM9B61f+Nk2tKoUmkvlCsTCuOVFAIrCcB1ItSxl7iRWgaPVRfLv2IWpT2sloOA7w955fb/l3Ws7f2AdTRWai4NU/dmfSmMr6J8z+59nipOH/MuCNuOWR/RAJL4JVL/4ndc6aOcF03t8htJ1qJi0duiy7nH9amSvayL8oihS5WPFMII29dcExqiZbZMKbQrsxDCorv8RozJNxeGlQEWka6lnPczjZMyLuVjj9SyzAoPqG5n96MhVxiRQZtv+DPb33BLfxWyuU1HJb3JRDNfdeA38D6Vk4sqV3dayNxLbHaaHV3mgA7zmu6yJ5GXkjFeI6Fzat4+Y8okplidQo8J2fjfib88fheKkz8BAAD//0R+RYg=" } From 14ea02ff7fea31c6601ca5f2d24c6259b18e5316 Mon Sep 17 00:00:00 2001 From: Mariana Date: Fri, 4 Oct 2019 13:45:05 +0200 Subject: [PATCH 08/21] Adding audit filesets --- filebeat/docs/fields.asciidoc | 225 +++++++++++++++++- filebeat/input/kafka/azure_log_patterns.go | 197 ++++++++++++++- filebeat/input/kafka/config.go | 34 +-- filebeat/input/kafka/input.go | 30 +++ .../activitylogs/config/activitylogs.yml | 4 +- .../module/azure/auditlogs/_meta/fields.yml | 5 + .../azure/auditlogs/config/auditlogs.yml | 14 ++ .../azure/auditlogs/ingest/pipeline.json | 43 ++++ .../module/azure/auditlogs/manifest.yml | 12 + x-pack/filebeat/module/azure/fields.go | 2 +- x-pack/filebeat/module/azure/module.yml | 4 +- .../module/azure/signinlogs/_meta/fields.yml | 5 + .../azure/signinlogs/config/signinlogs.yml | 14 ++ .../azure/signinlogs/ingest/pipeline.json | 43 ++++ .../module/azure/signinlogs/manifest.yml | 12 + 15 files changed, 616 insertions(+), 28 deletions(-) create mode 100644 x-pack/filebeat/module/azure/auditlogs/_meta/fields.yml create mode 100644 x-pack/filebeat/module/azure/auditlogs/config/auditlogs.yml create mode 100644 x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.json create mode 100644 x-pack/filebeat/module/azure/auditlogs/manifest.yml create mode 100644 x-pack/filebeat/module/azure/signinlogs/_meta/fields.yml create mode 100644 x-pack/filebeat/module/azure/signinlogs/config/signinlogs.yml create mode 100644 x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.json create mode 100644 x-pack/filebeat/module/azure/signinlogs/manifest.yml diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index aff526a6ec9..dfa82752f09 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -16,7 +16,6 @@ grouped in the following categories: * <> * <> * <> -* <> * <> * <> * <> @@ -1262,10 +1261,228 @@ azure Module Fields for azure Activity logs. -[[exported-fields-azure]] -== Decode azure processor fields fields -Common Event Format (azure) data. +[float] +=== identity + +The canonical user ID of the owner of the source bucket. + + + +*`azure.activitylogs.identity.claims.*`*:: ++ +-- +Claims + + +type: object + +-- + +[float] +=== authorization + +Node allocatable pods + + + +[float] +=== evidence + +Node allocatable pods + + + +*`azure.activitylogs.identity.authorization.evidence.roleAssignmentScope`*:: ++ +-- +Role assignment scope + + +type: keyword + +-- + +*`azure.activitylogs.identity.authorization.evidence.roleDefinitionId`*:: ++ +-- +Role definition ID + + +type: keyword + +-- + +*`azure.activitylogs.identity.authorization.evidence.role`*:: ++ +-- +Role + + +type: keyword + +-- + +*`azure.activitylogs.identity.authorization.evidence.roleAssignmentId`*:: ++ +-- +Role assignment ID + + +type: keyword + +-- + +*`azure.activitylogs.identity.authorization.evidence.principalId`*:: ++ +-- +Principal ID + + +type: keyword + +-- + +*`azure.activitylogs.identity.authorization.evidence.principalType`*:: ++ +-- +Principal type + + +type: keyword + +-- + +*`azure.activitylogs.identity.scope`*:: ++ +-- +Scope + + +type: keyword + +-- + +*`azure.activitylogs.identity.action`*:: ++ +-- +Action + + +type: keyword + +-- + +*`azure.activitylogs.correlationId`*:: ++ +-- +Correlation ID + + +type: keyword + +-- + +*`azure.activitylogs.resultType`*:: ++ +-- +Result Type + + +type: keyword + +-- + +*`azure.activitylogs.callerIpAddress`*:: ++ +-- +Caller Ip address + + +type: keyword + +-- + +*`azure.activitylogs.resourceID`*:: ++ +-- +Resource ID + + +type: keyword + +-- + +*`azure.activitylogs.level`*:: ++ +-- +Level + + +type: keyword + +-- + +*`azure.activitylogs.operationName`*:: ++ +-- +Operation name + + +type: keyword + +-- + +*`azure.activitylogs.resultSignature`*:: ++ +-- +Result signature + + +type: keyword + +-- + +*`azure.activitylogs.properties.*`*:: ++ +-- +Properties + + +type: object + +-- + +*`azure.activitylogs.location`*:: ++ +-- +Location + + +type: keyword + +-- + +*`azure.activitylogs.category`*:: ++ +-- +Category + + +type: keyword + +-- + +[float] +=== auditlogs + +Fields for azure audit logs. + + +[float] +=== signinlogs + +Fields for azure audit logs. [[exported-fields-beat-common]] diff --git a/filebeat/input/kafka/azure_log_patterns.go b/filebeat/input/kafka/azure_log_patterns.go index 1b2b45b5595..06d283def84 100644 --- a/filebeat/input/kafka/azure_log_patterns.go +++ b/filebeat/input/kafka/azure_log_patterns.go @@ -1,10 +1,28 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package kafka import "time" const ( - ActivityLogs = "ActivityLogs" - AuditLogs = "AuditLogs" + ActivityLogs = "ActivityLogs" + AuditLogs = "AuditLogs" + SigninLogs = "SigninLogs" ) // ActivityLogs structure matches the azure activity log format @@ -60,3 +78,178 @@ type ActivityLog struct { type AzureActivityLogs struct { Records []ActivityLog `json:"records"` } + +type AzureAuditLogs struct { + Records []AuditLog `json:"records"` +} + +type AzureSigninLogs struct { + Records []SigninLog `json:"records"` +} + +type AuditLog struct { + Category string `json:"category"` + CorrelationID string `json:"correlationId"` + DurationMs int `json:"durationMs"` + Level string `json:"level"` + OperationName string `json:"operationName"` + OperationVersion string `json:"operationVersion"` + Properties struct { + ActivityDateTime string `json:"activityDateTime"` + ActivityDisplayName string `json:"activityDisplayName"` + AdditionalDetails []interface{} `json:"additionalDetails"` + Category string `json:"category"` + CorrelationID string `json:"correlationId"` + ID string `json:"id"` + InitiatedBy struct { + App struct { + AppID interface{} `json:"appId"` + DisplayName string `json:"displayName"` + ServicePrincipalID string `json:"servicePrincipalId"` + ServicePrincipalName interface{} `json:"servicePrincipalName"` + } `json:"app"` + User struct { + DisplayName interface{} `json:"displayName"` + ID string `json:"id"` + IPAddress interface{} `json:"ipAddress"` + UserPrincipalName string `json:"userPrincipalName"` + } `json:"user"` + } `json:"initiatedBy"` + LoggedByService string `json:"loggedByService"` + OperationType string `json:"operationType"` + Result string `json:"result"` + ResultReason string `json:"resultReason"` + TargetResources []struct { + DisplayName interface{} `json:"displayName"` + ID string `json:"id"` + ModifiedProperties []struct { + DisplayName string `json:"displayName"` + NewValue string `json:"newValue"` + OldValue string `json:"oldValue"` + } `json:"modifiedProperties"` + Type string `json:"type"` + UserPrincipalName string `json:"userPrincipalName"` + } `json:"targetResources"` + } `json:"properties"` + ResourceID string `json:"resourceId"` + ResultSignature string `json:"resultSignature"` + TenantID string `json:"tenantId"` + Time string `json:"time"` + Identity string `json:"identity"` +} + +type SigninLog struct { + Level int `json:"Level"` + CallerIPAddress string `json:"callerIpAddress"` + Category string `json:"category"` + CorrelationID string `json:"correlationId"` + DurationMs int `json:"durationMs"` + Identity string `json:"identity"` + Location string `json:"location"` + OperationName string `json:"operationName"` + OperationVersion string `json:"operationVersion"` + Properties struct { + SignInBondData struct { + ConditionalAccessDetails interface{} `json:"ConditionalAccessDetails"` + DeviceDetails struct { + BrowserID interface{} `json:"BrowserId"` + BrowserType string `json:"BrowserType"` + DeviceDisplayName string `json:"DeviceDisplayName"` + DeviceID string `json:"DeviceId"` + DevicePlatform string `json:"DevicePlatform"` + DeviceTrustType int `json:"DeviceTrustType"` + IsCompliant interface{} `json:"IsCompliant"` + IsManaged interface{} `json:"IsManaged"` + UserAgent string `json:"UserAgent"` + } `json:"DeviceDetails"` + DisplayDetails struct { + ApplicationDisplayName string `json:"ApplicationDisplayName"` + AttemptedUsername interface{} `json:"AttemptedUsername"` + ProxyRestrictionTargetTenantName interface{} `json:"ProxyRestrictionTargetTenantName"` + ResourceDisplayName string `json:"ResourceDisplayName"` + UserName string `json:"UserName"` + } `json:"DisplayDetails"` + IssuerDetails interface{} `json:"IssuerDetails"` + LocationDetails struct { + IPChain interface{} `json:"IPChain"` + Latitude float64 `json:"Latitude"` + Longitude float64 `json:"Longitude"` + } `json:"LocationDetails"` + MfaDetails struct { + AuthMethod interface{} `json:"AuthMethod"` + MaskedDeviceID interface{} `json:"MaskedDeviceId"` + MfaStatus int `json:"MfaStatus"` + SasStatus interface{} `json:"SasStatus"` + } `json:"MfaDetails"` + PassThroughAuthenticationDetails interface{} `json:"PassThroughAuthenticationDetails"` + ProtocolDetails struct { + AuthenticationMethodsUsed interface{} `json:"AuthenticationMethodsUsed"` + DomainHintPresent interface{} `json:"DomainHintPresent"` + IsInteractive interface{} `json:"IsInteractive"` + LoginHintPresent interface{} `json:"LoginHintPresent"` + NetworkLocation interface{} `json:"NetworkLocation"` + Protocol interface{} `json:"Protocol"` + ResponseTime int `json:"ResponseTime"` + } `json:"ProtocolDetails"` + RAMDetails interface{} `json:"RamDetails"` + SourceAlpEvents interface{} `json:"SourceAlpEvents"` + } `json:"SignInBondData"` + AppDisplayName string `json:"appDisplayName"` + AppID string `json:"appId"` + AppliedConditionalAccessPolicies []interface{} `json:"appliedConditionalAccessPolicies"` + AuthenticationDetails []struct { + AuthenticationStepDateTime string `json:"authenticationStepDateTime"` + AuthenticationStepRequirement string `json:"authenticationStepRequirement"` + AuthenticationStepResultDetail string `json:"authenticationStepResultDetail"` + Succeeded bool `json:"succeeded"` + } `json:"authenticationDetails"` + AuthenticationProcessingDetails []interface{} `json:"authenticationProcessingDetails"` + AuthenticationRequirementPolicies []interface{} `json:"authenticationRequirementPolicies"` + ClientAppUsed string `json:"clientAppUsed"` + CorrelationID string `json:"correlationId"` + CreatedDateTime string `json:"createdDateTime"` + DeviceDetail struct { + DeviceID string `json:"deviceId"` + DisplayName string `json:"displayName"` + OperatingSystem string `json:"operatingSystem"` + TrustType string `json:"trustType"` + } `json:"deviceDetail"` + ID string `json:"id"` + IPAddress string `json:"ipAddress"` + IsInteractive bool `json:"isInteractive"` + Location struct { + City string `json:"city"` + CountryOrRegion string `json:"countryOrRegion"` + GeoCoordinates struct { + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` + } `json:"geoCoordinates"` + State string `json:"state"` + } `json:"location"` + MfaDetail struct{} `json:"mfaDetail"` + NetworkLocationDetails []interface{} `json:"networkLocationDetails"` + OriginalRequestID string `json:"originalRequestId"` + ProcessingTimeInMilliseconds int `json:"processingTimeInMilliseconds"` + ResourceDisplayName string `json:"resourceDisplayName"` + ResourceID string `json:"resourceId"` + RiskDetail string `json:"riskDetail"` + RiskEventTypes []interface{} `json:"riskEventTypes"` + RiskLevelAggregated string `json:"riskLevelAggregated"` + RiskLevelDuringSignIn string `json:"riskLevelDuringSignIn"` + RiskState string `json:"riskState"` + Status struct { + AdditionalDetails string `json:"additionalDetails"` + ErrorCode int `json:"errorCode"` + } `json:"status"` + TokenIssuerName string `json:"tokenIssuerName"` + TokenIssuerType string `json:"tokenIssuerType"` + UserDisplayName string `json:"userDisplayName"` + UserID string `json:"userId"` + UserPrincipalName string `json:"userPrincipalName"` + } `json:"properties"` + ResourceID string `json:"resourceId"` + ResultSignature string `json:"resultSignature"` + ResultType string `json:"resultType"` + TenantID string `json:"tenantId"` + Time string `json:"time"` +} diff --git a/filebeat/input/kafka/config.go b/filebeat/input/kafka/config.go index 4bf137ef69c..8d328bca3df 100644 --- a/filebeat/input/kafka/config.go +++ b/filebeat/input/kafka/config.go @@ -34,23 +34,23 @@ import ( type kafkaInputConfig struct { // Kafka hosts with port, e.g. "localhost:9092" - Hosts []string `config:"hosts" validate:"required"` - Topics []string `config:"topics" validate:"required"` - GroupID string `config:"group_id" validate:"required"` - ClientID string `config:"client_id"` - Version kafka.Version `config:"version"` - InitialOffset initialOffset `config:"initial_offset"` - ConnectBackoff time.Duration `config:"connect_backoff" validate:"min=0"` - ConsumeBackoff time.Duration `config:"consume_backoff" validate:"min=0"` - WaitClose time.Duration `config:"wait_close" validate:"min=0"` - MaxWaitTime time.Duration `config:"max_wait_time"` - IsolationLevel isolationLevel `config:"isolation_level"` - Fetch kafkaFetch `config:"fetch"` - Rebalance kafkaRebalance `config:"rebalance"` - TLS *tlscommon.Config `config:"ssl"` - Username string `config:"username"` - Password string `config:"password"` - AzureLogs string `config:"azure_logs"` + Hosts []string `config:"hosts" validate:"required"` + Topics []string `config:"topics" validate:"required"` + GroupID string `config:"group_id" validate:"required"` + ClientID string `config:"client_id"` + Version kafka.Version `config:"version"` + InitialOffset initialOffset `config:"initial_offset"` + ConnectBackoff time.Duration `config:"connect_backoff" validate:"min=0"` + ConsumeBackoff time.Duration `config:"consume_backoff" validate:"min=0"` + WaitClose time.Duration `config:"wait_close" validate:"min=0"` + MaxWaitTime time.Duration `config:"max_wait_time"` + IsolationLevel isolationLevel `config:"isolation_level"` + Fetch kafkaFetch `config:"fetch"` + Rebalance kafkaRebalance `config:"rebalance"` + TLS *tlscommon.Config `config:"ssl"` + Username string `config:"username"` + Password string `config:"password"` + AzureLogs string `config:"azure_logs"` } type kafkaFetch struct { diff --git a/filebeat/input/kafka/input.go b/filebeat/input/kafka/input.go index 145965bcce4..ff431408758 100644 --- a/filebeat/input/kafka/input.go +++ b/filebeat/input/kafka/input.go @@ -335,6 +335,22 @@ func (h *groupHandler) parseMultipleMessages(bMessage []byte) []string { case "": default: return []string{string(bMessage)} + case AuditLogs: + // if the fileset is audit logs a filtering of the messages should be done as the eventhub can return different types of messages + var obj AzureAuditLogs + test := string(bMessage) + _ = test + err := json.Unmarshal(bMessage, &obj) + if err != nil { + return nil + } + for _, ms := range obj.Records { + js, err := json.Marshal(ms) + if err == nil { + messages = append(messages, string(js)) + } + } + return messages case ActivityLogs: // if the fileset is activity logs a filtering of the messages should be done as the eventhub can return different types of messages var obj AzureActivityLogs @@ -349,6 +365,20 @@ func (h *groupHandler) parseMultipleMessages(bMessage []byte) []string { } } return messages + case SigninLogs: + // if the fileset is signin logs a filtering of the messages should be done as the eventhub can return different types of messages + var obj AzureSigninLogs + err := json.Unmarshal(bMessage, &obj) + if err != nil { + return nil + } + for _, ms := range obj.Records { + js, err := json.Marshal(ms) + if err == nil { + messages = append(messages, string(js)) + } + } + return messages } return nil } diff --git a/x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml b/x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml index ebe34ec771c..bc3e31c0c16 100644 --- a/x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml +++ b/x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml @@ -1,6 +1,6 @@ type: azure -hosts: {{ .eventhubs_namespace }} -topics: {{ .topics }} +hosts: {{ .namespace }} +topics: {{ .eventhub }} group_id: {{ .consumer_group }} azure_logs: "ActivityLogs" diff --git a/x-pack/filebeat/module/azure/auditlogs/_meta/fields.yml b/x-pack/filebeat/module/azure/auditlogs/_meta/fields.yml new file mode 100644 index 00000000000..30ebf2a29fd --- /dev/null +++ b/x-pack/filebeat/module/azure/auditlogs/_meta/fields.yml @@ -0,0 +1,5 @@ +- name: auditlogs + type: group + description: > + Fields for azure audit logs. + fields: diff --git a/x-pack/filebeat/module/azure/auditlogs/config/auditlogs.yml b/x-pack/filebeat/module/azure/auditlogs/config/auditlogs.yml new file mode 100644 index 00000000000..af0b1fb1dd4 --- /dev/null +++ b/x-pack/filebeat/module/azure/auditlogs/config/auditlogs.yml @@ -0,0 +1,14 @@ +type: kafka +hosts: {{ .namespace }} +topics: {{ .eventhub }} +group_id: {{ .consumer_group }} +azure_logs: "AuditLogs" + +username: "$ConnectionString" +password: {{ .connection_string }} +ssl.enabled: true + +processors: +- decode_json_fields: + fields: ["message"] + target: "" diff --git a/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.json b/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.json new file mode 100644 index 00000000000..f394b9cece5 --- /dev/null +++ b/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.json @@ -0,0 +1,43 @@ +{ + "description": "Pipeline for parsing azure activity logs.", + "processors": [ + { + "date": { + "field": "time", + "target_field": "@timestamp", + "ignore_failure": false, + "formats": [ + "ISO8601" + ] + } + }, + { + "rename": { + "field": "resourceId", + "target_field": "resourceID", + "ignore_missing": true + } + }, + { + "rename": { + "field": "identity", + "target_field": "specification", + "ignore_missing": true + } + }, + { + "remove": { + "field": ["message", "time"], + "ignore_missing": true + } + } + ], + "on_failure": [ + { + "set": { + "field": "error.message", + "value": "{{ _ingest.on_failure_message }}" + } + } + ] +} diff --git a/x-pack/filebeat/module/azure/auditlogs/manifest.yml b/x-pack/filebeat/module/azure/auditlogs/manifest.yml new file mode 100644 index 00000000000..9e06c553951 --- /dev/null +++ b/x-pack/filebeat/module/azure/auditlogs/manifest.yml @@ -0,0 +1,12 @@ +module_version: 1.0 + +var: + - name: input + default: kafka + - name: topics + default: "insights-logs-auditlogs" + - name: consumer_group + default: "$Default" + +ingest_pipeline: ingest/pipeline.json +input: config/auditlogs.yml diff --git a/x-pack/filebeat/module/azure/fields.go b/x-pack/filebeat/module/azure/fields.go index c7539a97f28..d722070c700 100644 --- a/x-pack/filebeat/module/azure/fields.go +++ b/x-pack/filebeat/module/azure/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAzure returns asset data. // This is the base64 encoded gzipped contents of module/azure. func AssetAzure() string { - return "eJzEl81u4jAQx+88xYhjpfIAHFZCRSsh7XartvfK2EPwYjyW7VClT79yIB8Q54MNy/oWPPn/f2OPneERdpjNgX2lFicAXnqFc5jmz9MJgEDHrTRekp7DtwkAHGPhJ4lUhVc2EpVw83zqETTbYyUXhs8MziGxlJrTLxHNc5kzKe7lQfpMUeLKyZhqq/JxfM/1YUP2lMDiJAxBeVaLvSSp00iB2kufnU220fQQhfG+ReBMk5acKUgdWlgtgTbgtwj0qdEWD45SyxHWKd+hn10IxZDr2FwxuXezh0ZAgU7r38h9ZPo48XGM2mH2SVZ0h33smTFSJ6d3pg/TSHzPsoTxlDO3psRSvyUrv1jQaM0rtiUD/Z9JIDCliDPP1grBkGjitK99HRYPoXA4RoP6WAfyXsPcx11nt6Rw4ZxM9B61f+Nk2tKoUmkvlCsTCuOVFAIrCcB1ItSxl7iRWgaPVRfLv2IWpT2sloOA7w955fb/l3Ws7f2AdTRWai4NU/dmfSmMr6J8z+59nipOH/MuCNuOWR/RAJL4JVL/4ndc6aOcF03t8htJ1qJi0duiy7nH9amSvayL8oihS5WPFMII29dcExqiZbZMKbQrsxDCorv8RozJNxeGlQEWka6lnPczjZMyLuVjj9SyzAoPqG5n96MhVxiRQZtv+DPb33BLfxWyuU1HJb3JRDNfdeA38D6Vk4sqV3dayNxLbHaaHV3mgA7zmu6yJ5GXkjFeI6Fzat4+Y8okplidQo8J2fjfib88fheKkz8BAAD//0R+RYg=" + return "eJzEl91uIiEUx+99ihMvm9QH8GITU7OJyW63aXvfIBzHsyIQYGymT79hdD50mA93XJc7hfn/fwcOcHiEHWZzYF+pxQmAJy9xDtP893QCINBxS8aTVnP4NgGA41j4qUUqwycbQincPO96BMX2WMmF5jODc0isTs3pn4jmucyZFPd0IJ9JnbiyM6baqnxs33N92Gh7CmBxEoagPKuNvSSp05BA5clnZ51tND1Eob1vEThTWhFnElKHFlZL0BvwWwT9qdAWP5xOLUdYp3yHfnYhFEOuY3PJaO9mD40BBbpe/0buI93Hjo/jqB1mn9qK7mEfe2YMqeT0zfRhGhnfMy2hPeXMrSGx1G+1pS8WNFrjii3JQP9nLRCYlJozz9YSwWjRxGmf+zosHkLicIwO6mMdyHsNcx93nd1qiQvnKFF7VP6Na9MWRhVKe6JcGVBor1oisJIAXCdCHXuJG1IUPFZdLP+KWZT2sFoOAr4/5JXL/1/msbb2A+bRWFKcDJP3Zn0pjK+ifM/uvZ8qTh/zLgjbtlkf0QCS+CFSv/E7jvRRzoumdnlHamtRsuhp0eXc4/pUyV7mRbnF0KXSRxJhhO1rrgkN0TJaJiXalVkIYdFd3hFj4s2FYWWARaRrIef1TGOnjAv5WCO1TLPEA8rb2f1oyBVG2qDNF/yZ7W+4pL8K2dymI5PeKFHMVxX4DbxP6eSiytWZFiL3hM1Ks6PKHFBhXlNd9gTyUjLGcyRUTs3TZ0yaxBSrXegx0Tb+nPjL7Xeh2HxQpYL8zV9TuWrvU6q8WyhRpO4M8ScAAP//XRKL7w==" } diff --git a/x-pack/filebeat/module/azure/module.yml b/x-pack/filebeat/module/azure/module.yml index 3d473aed6c4..d71d79b7561 100644 --- a/x-pack/filebeat/module/azure/module.yml +++ b/x-pack/filebeat/module/azure/module.yml @@ -1,3 +1,3 @@ dashboards: -- id: Filebeat-azure-activitylogs-Dashboard - file: Filebeat-azure-activitylogs.json +- id: Filebeat-azure-auditlogs-Dashboard + file: Filebeat-azure-audit.json diff --git a/x-pack/filebeat/module/azure/signinlogs/_meta/fields.yml b/x-pack/filebeat/module/azure/signinlogs/_meta/fields.yml new file mode 100644 index 00000000000..7b5d583554a --- /dev/null +++ b/x-pack/filebeat/module/azure/signinlogs/_meta/fields.yml @@ -0,0 +1,5 @@ +- name: signinlogs + type: group + description: > + Fields for azure audit logs. + fields: diff --git a/x-pack/filebeat/module/azure/signinlogs/config/signinlogs.yml b/x-pack/filebeat/module/azure/signinlogs/config/signinlogs.yml new file mode 100644 index 00000000000..7a258346faf --- /dev/null +++ b/x-pack/filebeat/module/azure/signinlogs/config/signinlogs.yml @@ -0,0 +1,14 @@ +type: kafka +hosts: {{ .namespace }} +topics: {{ .eventhub }} +group_id: {{ .consumer_group }} +azure_logs: "SigninLogs" + +username: "$ConnectionString" +password: {{ .connection_string }} +ssl.enabled: true + +processors: +- decode_json_fields: + fields: ["message"] + target: "" diff --git a/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.json b/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.json new file mode 100644 index 00000000000..08a42caa448 --- /dev/null +++ b/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.json @@ -0,0 +1,43 @@ +{ + "description": "Pipeline for parsing azure signin logs.", + "processors": [ + { + "date": { + "field": "time", + "target_field": "@timestamp", + "ignore_failure": false, + "formats": [ + "ISO8601" + ] + } + }, + { + "rename": { + "field": "resourceId", + "target_field": "resourceID", + "ignore_missing": true + } + }, + { + "rename": { + "field": "identity", + "target_field": "specification", + "ignore_missing": true + } + }, + { + "remove": { + "field": ["message", "time"], + "ignore_missing": true + } + } + ], + "on_failure": [ + { + "set": { + "field": "error.message", + "value": "{{ _ingest.on_failure_message }}" + } + } + ] +} diff --git a/x-pack/filebeat/module/azure/signinlogs/manifest.yml b/x-pack/filebeat/module/azure/signinlogs/manifest.yml new file mode 100644 index 00000000000..5ba56ad8df3 --- /dev/null +++ b/x-pack/filebeat/module/azure/signinlogs/manifest.yml @@ -0,0 +1,12 @@ +module_version: 1.0 + +var: + - name: input + default: kafka + - name: topics + default: "insights-logs-auditlogs" + - name: consumer_group + default: "$Default" + +ingest_pipeline: ingest/pipeline.json +input: config/signinlogs.yml From 75ec856b0bb10f5a058779751f64064b24a2bb09 Mon Sep 17 00:00:00 2001 From: Mariana Date: Fri, 4 Oct 2019 15:16:43 +0200 Subject: [PATCH 09/21] work on kafka input --- filebeat/docs/modules/azure.asciidoc | 2 +- filebeat/input/kafka/config.go | 4 +-- filebeat/input/kafka/input.go | 4 +-- x-pack/filebeat/include/list.go | 1 - x-pack/filebeat/input/azure/config.go | 14 --------- x-pack/filebeat/input/azure/input.go | 31 ------------------- .../filebeat/module/azure/_meta/docs.asciidoc | 2 +- .../activitylogs/config/activitylogs.yml | 2 +- .../module/azure/activitylogs/manifest.yml | 2 +- .../module/azure/auditlogs/manifest.yml | 2 +- .../module/azure/signinlogs/manifest.yml | 2 +- 11 files changed, 10 insertions(+), 56 deletions(-) delete mode 100644 x-pack/filebeat/input/azure/config.go delete mode 100644 x-pack/filebeat/input/azure/input.go diff --git a/filebeat/docs/modules/azure.asciidoc b/filebeat/docs/modules/azure.asciidoc index 3251b267668..9c6d77389f0 100644 --- a/filebeat/docs/modules/azure.asciidoc +++ b/filebeat/docs/modules/azure.asciidoc @@ -4,7 +4,7 @@ This file is generated! See scripts/docs_collector.py [[filebeat-module-azure]] :modulename: azure -:has-dashboards: true +:has-dashboards: false == azure module diff --git a/filebeat/input/kafka/config.go b/filebeat/input/kafka/config.go index 8d328bca3df..8e9a34cfd07 100644 --- a/filebeat/input/kafka/config.go +++ b/filebeat/input/kafka/config.go @@ -104,7 +104,7 @@ var ( // The default config for the kafka input. When in doubt, default values // were chosen to match sarama's defaults. -func DefaultConfig() kafkaInputConfig { +func defaultConfig() kafkaInputConfig { return kafkaInputConfig{ Version: kafka.Version("1.0.0"), InitialOffset: initialOffsetOldest, @@ -144,7 +144,7 @@ func (c *kafkaInputConfig) Validate() error { return nil } -func NewSaramaConfig(config kafkaInputConfig) (*sarama.Config, error) { +func newSaramaConfig(config kafkaInputConfig) (*sarama.Config, error) { k := sarama.NewConfig() version, ok := config.Version.Get() diff --git a/filebeat/input/kafka/input.go b/filebeat/input/kafka/input.go index ff431408758..f16ee18da70 100644 --- a/filebeat/input/kafka/input.go +++ b/filebeat/input/kafka/input.go @@ -63,7 +63,7 @@ func NewInput( inputContext input.Context, ) (input.Input, error) { - config := DefaultConfig() + config := defaultConfig() if err := cfg.Unpack(&config); err != nil { return nil, errors.Wrap(err, "reading kafka input config") } @@ -86,7 +86,7 @@ func NewInput( return nil, err } - saramaConfig, err := NewSaramaConfig(config) + saramaConfig, err := newSaramaConfig(config) if err != nil { return nil, errors.Wrap(err, "initializing Sarama config") } diff --git a/x-pack/filebeat/include/list.go b/x-pack/filebeat/include/list.go index b672f30181b..401e4226c48 100644 --- a/x-pack/filebeat/include/list.go +++ b/x-pack/filebeat/include/list.go @@ -8,7 +8,6 @@ package include import ( // Import packages that need to register themselves. - _ "github.com/elastic/beats/x-pack/filebeat/input/azure" _ "github.com/elastic/beats/x-pack/filebeat/input/googlepubsub" _ "github.com/elastic/beats/x-pack/filebeat/input/netflow" _ "github.com/elastic/beats/x-pack/filebeat/input/s3" diff --git a/x-pack/filebeat/input/azure/config.go b/x-pack/filebeat/input/azure/config.go deleted file mode 100644 index 1a9a3332d1c..00000000000 --- a/x-pack/filebeat/input/azure/config.go +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package azure - -var defaultConfig = config{ - YieldEventsFromField: "records", -} - -type config struct { - // YieldEventsFromField (i) - YieldEventsFromField string `config:"yield_events_from_field"` -} diff --git a/x-pack/filebeat/input/azure/input.go b/x-pack/filebeat/input/azure/input.go deleted file mode 100644 index cb6a5cbed30..00000000000 --- a/x-pack/filebeat/input/azure/input.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package azure - -import ( - "github.com/elastic/beats/filebeat/input/kafka" - - "github.com/elastic/beats/filebeat/channel" - "github.com/elastic/beats/filebeat/input" - "github.com/elastic/beats/libbeat/common" -) - -func init() { - err := input.Register("azure", NewInput) - if err != nil { - panic(err) - } -} - -// NewInput creates a new kafka input -func NewInput( - cfg *common.Config, - connector channel.Connector, - context input.Context, -) (input.Input, error) { - // Wrap log input with custom docker settings - - return kafka.NewInput(cfg, connector, context) -} diff --git a/x-pack/filebeat/module/azure/_meta/docs.asciidoc b/x-pack/filebeat/module/azure/_meta/docs.asciidoc index 6e68e7da183..73af955965b 100644 --- a/x-pack/filebeat/module/azure/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/azure/_meta/docs.asciidoc @@ -1,5 +1,5 @@ :modulename: azure -:has-dashboards: true +:has-dashboards: false == azure module diff --git a/x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml b/x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml index bc3e31c0c16..2a95c87cb14 100644 --- a/x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml +++ b/x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml @@ -1,4 +1,4 @@ -type: azure +type: kafka hosts: {{ .namespace }} topics: {{ .eventhub }} group_id: {{ .consumer_group }} diff --git a/x-pack/filebeat/module/azure/activitylogs/manifest.yml b/x-pack/filebeat/module/azure/activitylogs/manifest.yml index e85d461325e..d2bf93f2ce5 100644 --- a/x-pack/filebeat/module/azure/activitylogs/manifest.yml +++ b/x-pack/filebeat/module/azure/activitylogs/manifest.yml @@ -2,7 +2,7 @@ module_version: 1.0 var: - name: input - default: azure + default: kafka - name: topics default: "insights-operational-logs" - name: consumer_group diff --git a/x-pack/filebeat/module/azure/auditlogs/manifest.yml b/x-pack/filebeat/module/azure/auditlogs/manifest.yml index 9e06c553951..f74e0f3593e 100644 --- a/x-pack/filebeat/module/azure/auditlogs/manifest.yml +++ b/x-pack/filebeat/module/azure/auditlogs/manifest.yml @@ -3,7 +3,7 @@ module_version: 1.0 var: - name: input default: kafka - - name: topics + - name: eventhub default: "insights-logs-auditlogs" - name: consumer_group default: "$Default" diff --git a/x-pack/filebeat/module/azure/signinlogs/manifest.yml b/x-pack/filebeat/module/azure/signinlogs/manifest.yml index 5ba56ad8df3..785bc7e01c1 100644 --- a/x-pack/filebeat/module/azure/signinlogs/manifest.yml +++ b/x-pack/filebeat/module/azure/signinlogs/manifest.yml @@ -3,7 +3,7 @@ module_version: 1.0 var: - name: input default: kafka - - name: topics + - name: eventhub default: "insights-logs-auditlogs" - name: consumer_group default: "$Default" From 8f156ea3fa236cf195e7da932d7b5fed5ad5eca7 Mon Sep 17 00:00:00 2001 From: Mariana Date: Fri, 4 Oct 2019 16:27:39 +0200 Subject: [PATCH 10/21] Fix config --- filebeat/docs/modules/azure.asciidoc | 2 ++ x-pack/filebeat/filebeat.reference.yml | 22 ++++++++++++++----- x-pack/filebeat/module/azure/_meta/config.yml | 22 ++++++++++++++----- .../filebeat/module/azure/_meta/docs.asciidoc | 2 ++ x-pack/filebeat/modules.d/azure.yml.disabled | 22 ++++++++++++++----- 5 files changed, 55 insertions(+), 15 deletions(-) diff --git a/filebeat/docs/modules/azure.asciidoc b/filebeat/docs/modules/azure.asciidoc index 9c6d77389f0..067f670d188 100644 --- a/filebeat/docs/modules/azure.asciidoc +++ b/filebeat/docs/modules/azure.asciidoc @@ -8,6 +8,8 @@ This file is generated! See scripts/docs_collector.py == azure module +beta[] + This is the azure module. include::../include/what-happens.asciidoc[] diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 5e4c05d67de..65aa179eb79 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -93,14 +93,26 @@ filebeat.modules: activitylogs: enabled: true var: - eventhubs_namespace: "" - topics: ["insights-operational-logs"] + namespace: "" + eventhub: ["insights-operational-logs"] consumer_group: "$Default" connection_string: "" - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: + # auditlogs: + # enabled: true + # var: + # namespace: "" + # eventhub: ["insights-logs-auditlogs"] + # consumer_group: "$Default" + # connection_string: "" + + # signinlogs: + # enabled: true + # var: + # namespace: "" + # eventhub: ["insights-logs-signinlogs"] + # consumer_group: "$Default" + # connection_string: "" #--------------------------------- CEF Module --------------------------------- - module: cef diff --git a/x-pack/filebeat/module/azure/_meta/config.yml b/x-pack/filebeat/module/azure/_meta/config.yml index 9d77c6fbb4c..d8b7689d285 100644 --- a/x-pack/filebeat/module/azure/_meta/config.yml +++ b/x-pack/filebeat/module/azure/_meta/config.yml @@ -3,11 +3,23 @@ activitylogs: enabled: true var: - eventhubs_namespace: "" - topics: ["insights-operational-logs"] + namespace: "" + eventhub: ["insights-operational-logs"] consumer_group: "$Default" connection_string: "" - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: + # auditlogs: + # enabled: true + # var: + # namespace: "" + # eventhub: ["insights-logs-auditlogs"] + # consumer_group: "$Default" + # connection_string: "" + + # signinlogs: + # enabled: true + # var: + # namespace: "" + # eventhub: ["insights-logs-signinlogs"] + # consumer_group: "$Default" + # connection_string: "" diff --git a/x-pack/filebeat/module/azure/_meta/docs.asciidoc b/x-pack/filebeat/module/azure/_meta/docs.asciidoc index 73af955965b..cd17bfc2358 100644 --- a/x-pack/filebeat/module/azure/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/azure/_meta/docs.asciidoc @@ -3,6 +3,8 @@ == azure module +beta[] + This is the azure module. include::../include/what-happens.asciidoc[] diff --git a/x-pack/filebeat/modules.d/azure.yml.disabled b/x-pack/filebeat/modules.d/azure.yml.disabled index 2a0154535b5..e94149040a9 100644 --- a/x-pack/filebeat/modules.d/azure.yml.disabled +++ b/x-pack/filebeat/modules.d/azure.yml.disabled @@ -6,11 +6,23 @@ activitylogs: enabled: true var: - eventhubs_namespace: "" - topics: ["insights-operational-logs"] + namespace: "" + eventhub: ["insights-operational-logs"] consumer_group: "$Default" connection_string: "" - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: + # auditlogs: + # enabled: true + # var: + # namespace: "" + # eventhub: ["insights-logs-auditlogs"] + # consumer_group: "$Default" + # connection_string: "" + + # signinlogs: + # enabled: true + # var: + # namespace: "" + # eventhub: ["insights-logs-signinlogs"] + # consumer_group: "$Default" + # connection_string: "" From c04a47632871aab730a56085f3a267e436e6b20f Mon Sep 17 00:00:00 2001 From: Mariana Date: Mon, 7 Oct 2019 18:34:03 +0200 Subject: [PATCH 11/21] Work on defining fields and examples --- CHANGELOG.next.asciidoc | 1 + filebeat/docs/fields.asciidoc | 853 +++++++++++++++++- filebeat/docs/modules/azure.asciidoc | 86 +- filebeat/input/kafka/azure_log_patterns.go | 36 +- filebeat/input/kafka/input.go | 3 +- .../filebeat/module/azure/_meta/docs.asciidoc | 86 +- .../azure/activitylogs/test/activity_log.json | 95 ++ .../test/activity_log_expected.json | 103 +++ .../module/azure/auditlogs/_meta/fields.yml | 170 ++++ .../azure/auditlogs/test/audit_log.json | 78 ++ .../auditlogs/test/audit_log_expected.json | 107 +++ x-pack/filebeat/module/azure/fields.go | 2 +- .../module/azure/signinlogs/_meta/fields.yml | 198 ++++ .../azure/signinlogs/test/signin_log.json | 155 ++++ .../test/signin_log_expected.json} | 0 15 files changed, 1916 insertions(+), 57 deletions(-) create mode 100644 x-pack/filebeat/module/azure/activitylogs/test/activity_log.json create mode 100644 x-pack/filebeat/module/azure/auditlogs/test/audit_log.json create mode 100644 x-pack/filebeat/module/azure/auditlogs/test/audit_log_expected.json create mode 100644 x-pack/filebeat/module/azure/signinlogs/test/signin_log.json rename x-pack/filebeat/module/azure/{activitylogs/test/activityLog.json => signinlogs/test/signin_log_expected.json} (100%) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 2eaf3d282e2..0704bd32ed2 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -311,6 +311,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add module for ingesting Cisco FTD logs over syslog. {pull}13286[13286] - Update CoreDNS module to populate ECS DNS fields. {issue}13320[13320] {pull}13505[13505] - Parse query steps in PostgreSQL slowlogs. {issue}13496[13496] {pull}13701[13701] +- Add azure module. {pull}13776[13776] *Heartbeat* diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index dfa82752f09..9d1023ab914 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -1479,11 +1479,860 @@ type: keyword Fields for azure audit logs. + +*`azure.auditlogs.resourceID`*:: ++ +-- +Resource ID + + +type: keyword + +-- + +*`azure.auditlogs.operationName`*:: ++ +-- +The operation name + + +type: keyword + +-- + +*`azure.auditlogs.operationVersion`*:: ++ +-- +The operation version + + +type: keyword + +-- + +*`azure.auditlogs.category`*:: ++ +-- +Log category + + +type: keyword + +-- + +*`azure.auditlogs.tenantId`*:: ++ +-- +Tenant ID + + +type: keyword + +-- + +*`azure.auditlogs.resultSignature`*:: ++ +-- +Result signature + + +type: keyword + +-- + +*`azure.auditlogs.durationMs`*:: ++ +-- +Activity duration + + +type: keyword + +-- + +*`azure.auditlogs.correlationId`*:: ++ +-- +Correlation ID + + +type: keyword + +-- + +*`azure.auditlogs.level`*:: ++ +-- +Log level + + +type: keyword + +-- + +[float] +=== properties + +The audit log properties + + + +*`azure.auditlogs.properties.result`*:: ++ +-- +Log result + + +type: keyword + +-- + +*`azure.auditlogs.properties.activityDisplayName`*:: ++ +-- +Activity display name + + +type: keyword + +-- + +*`azure.auditlogs.properties.resultReason`*:: ++ +-- +Reason for the log result + + +type: keyword + +-- + +*`azure.auditlogs.properties.correlationId`*:: ++ +-- +Correlation ID + + +type: keyword + +-- + +*`azure.auditlogs.properties.loggedByService`*:: ++ +-- +Logged by service + + +type: keyword + +-- + +*`azure.auditlogs.properties.operationType`*:: ++ +-- +Operation type + + +type: keyword + +-- + +*`azure.auditlogs.properties.id`*:: ++ +-- +ID + + +type: keyword + +-- + +*`azure.auditlogs.properties.activityDateTime`*:: ++ +-- +Activity timestamp + + +type: keyword + +-- + +*`azure.auditlogs.properties.category`*:: ++ +-- +category + + +type: keyword + +-- + +[float] +=== targetResources + +Target resources + + + +*`azure.auditlogs.properties.targetResources.displayName`*:: ++ +-- +Display name + + +type: keyword + +-- + +*`azure.auditlogs.properties.targetResources.id`*:: ++ +-- +ID + + +type: keyword + +-- + +*`azure.auditlogs.properties.targetResources.type`*:: ++ +-- +Type + + +type: keyword + +-- + +*`azure.auditlogs.properties.targetResources.ipAddress`*:: ++ +-- +ip Address + + +type: keyword + +-- + +*`azure.auditlogs.properties.targetResources.userPrincipalName`*:: ++ +-- +User principal name + + +type: keyword + +-- + +[float] +=== modifiedProperties + +Modified properties + + + +*`azure.auditlogs.properties.targetResources.modifiedProperties.newValue`*:: ++ +-- +New value + + +type: keyword + +-- + +*`azure.auditlogs.properties.targetResources.modifiedProperties.displayName`*:: ++ +-- +Display value + + +type: keyword + +-- + +*`azure.auditlogs.properties.targetResources.modifiedProperties.oldValue`*:: ++ +-- +Old value + + +type: keyword + +-- + +[float] +=== initiatedBy + +Information regarding the initiator + + + +[float] +=== app + +App + + + +*`azure.auditlogs.properties.initiatedBy.app.servicePrincipalName`*:: ++ +-- +Service principal name + + +type: keyword + +-- + +*`azure.auditlogs.properties.initiatedBy.app.displayName`*:: ++ +-- +Display name + + +type: keyword + +-- + +*`azure.auditlogs.properties.initiatedBy.app.appId`*:: ++ +-- +App ID + + +type: keyword + +-- + +*`azure.auditlogs.properties.initiatedBy.app.servicePrincipalId`*:: ++ +-- +Service principal ID + + +type: keyword + +-- + +[float] +=== user + +User + + + +*`azure.auditlogs.properties.initiatedBy.user.userPrincipalName`*:: ++ +-- +User principal name + + +type: keyword + +-- + +*`azure.auditlogs.properties.initiatedBy.user.displayName`*:: ++ +-- +Display name + + +type: keyword + +-- + +*`azure.auditlogs.properties.initiatedBy.user.id`*:: ++ +-- +ID + + +type: keyword + +-- + +*`azure.auditlogs.properties.initiatedBy.user.ipAddress`*:: ++ +-- +ip Address + + +type: keyword + +-- + +[float] +=== signinlogs + +Fields for azure audit logs. + + + +*`azure.signinlogs.resourceID`*:: ++ +-- +Resource ID + + +type: keyword + +-- + +*`azure.signinlogs.operationName`*:: ++ +-- +The operation name + + +type: keyword + +-- + +*`azure.signinlogs.operationVersion`*:: ++ +-- +The operation version + + +type: keyword + +-- + +*`azure.signinlogs.category`*:: ++ +-- +Log category + + +type: keyword + +-- + +*`azure.signinlogs.tenantId`*:: ++ +-- +Tenant ID + + +type: keyword + +-- + +*`azure.signinlogs.resultSignature`*:: ++ +-- +Result signature + + +type: keyword + +-- + +*`azure.signinlogs.durationMs`*:: ++ +-- +Activity duration + + +type: keyword + +-- + +*`azure.signinlogs.correlationId`*:: ++ +-- +Correlation ID + + +type: keyword + +-- + +*`azure.signinlogs.level`*:: ++ +-- +Log level + + +type: keyword + +-- + +*`azure.signinlogs.resultType`*:: ++ +-- +Result type + + +type: keyword + +-- + +*`azure.signinlogs.callerIpAddress`*:: ++ +-- +Caller Ip address + + +type: keyword + +-- + +*`azure.signinlogs.identity`*:: ++ +-- +Identity + + +type: keyword + +-- + +*`azure.signinlogs.location`*:: ++ +-- +Location + + +type: keyword + +-- + +[float] +=== properties + +The signin log properties + + + +*`azure.signinlogs.properties.id`*:: ++ +-- +ID + + +type: keyword + +-- + +*`azure.signinlogs.properties.createdDateTime`*:: ++ +-- +Created date time + + +type: keyword + +-- + +*`azure.signinlogs.properties.userDisplayName`*:: ++ +-- +User display name + + +type: keyword + +-- + +*`azure.signinlogs.properties.correlationId`*:: ++ +-- +Correlation ID + + +type: keyword + +-- + +*`azure.signinlogs.properties.userPrincipalName`*:: ++ +-- +User principal name + + +type: keyword + +-- + +*`azure.signinlogs.properties.userId`*:: ++ +-- +User ID + + +type: keyword + +-- + +*`azure.signinlogs.properties.appId`*:: ++ +-- +App ID + + +type: keyword + +-- + +*`azure.signinlogs.properties.appDisplayName`*:: ++ +-- +App display name + + +type: keyword + +-- + +*`azure.signinlogs.properties.ipAddress`*:: ++ +-- +Ip address + + +type: keyword + +-- + +*`azure.signinlogs.properties.clientAppUsed`*:: ++ +-- +Client app used + + +type: keyword + +-- + +*`azure.signinlogs.properties.conditionalAccessStatus`*:: ++ +-- +Conditional access status + + +type: keyword + +-- + +*`azure.signinlogs.properties.originalRequestId`*:: ++ +-- +Original request ID + + +type: keyword + +-- + +*`azure.signinlogs.properties.isInteractive`*:: ++ +-- +Is interactive + + +type: keyword + +-- + +*`azure.signinlogs.properties.tokenIssuerName`*:: ++ +-- +Token issuer name + + +type: keyword + +-- + +*`azure.signinlogs.properties.tokenIssuerType`*:: ++ +-- +Token issuer type + + +type: keyword + +-- + +*`azure.signinlogs.properties.processingTimeInMilliseconds`*:: ++ +-- +Processing time in milliseconds + + +type: keyword + +-- + +*`azure.signinlogs.properties.riskDetail`*:: ++ +-- +Risk detail + + +type: keyword + +-- + +*`azure.signinlogs.properties.riskLevelAggregated`*:: ++ +-- +Risk level aggregated + + +type: keyword + +-- + +*`azure.signinlogs.properties.riskLevelDuringSignIn`*:: ++ +-- +Risk level during signIn + + +type: keyword + +-- + +*`azure.signinlogs.properties.riskState`*:: ++ +-- +Risk state + + +type: keyword + +-- + +*`azure.signinlogs.properties.resourceDisplayName`*:: ++ +-- +Resource display name + + +type: keyword + +-- + [float] -=== signinlogs +=== status + +Status + + + +*`azure.signinlogs.properties.status.errorCode`*:: ++ +-- +Error code + + +type: keyword + +-- + +[float] +=== deviceDetail + +Status + + + +*`azure.signinlogs.properties.deviceDetail.deviceId`*:: ++ +-- +Device ID + + +type: keyword + +-- + +*`azure.signinlogs.properties.deviceDetail.operatingSystem`*:: ++ +-- +Operating system + + +type: keyword + +-- + +*`azure.signinlogs.properties.deviceDetail.browser`*:: ++ +-- +Browser + + +type: keyword + +-- + +[float] +=== location + +Status + + + +*`azure.signinlogs.properties.location.city`*:: ++ +-- +City + + +type: keyword + +-- + +*`azure.signinlogs.properties.location.state`*:: ++ +-- +State -Fields for azure audit logs. +type: keyword + +-- + +*`azure.signinlogs.properties.location.countryOrRegion`*:: ++ +-- +Country or region + + +type: keyword + +-- + +[float] +=== geoCoordinates + +GeoCoordinates + + + +*`azure.signinlogs.properties.location.geoCoordinates.latitude`*:: ++ +-- +Latitude + + +type: keyword + +-- + +*`azure.signinlogs.properties.location.geoCoordinates.longitude`*:: ++ +-- +Longitude + + +type: keyword + +-- [[exported-fields-beat-common]] == Beat fields diff --git a/filebeat/docs/modules/azure.asciidoc b/filebeat/docs/modules/azure.asciidoc index 067f670d188..9b51551e2e0 100644 --- a/filebeat/docs/modules/azure.asciidoc +++ b/filebeat/docs/modules/azure.asciidoc @@ -12,6 +12,73 @@ beta[] This is the azure module. +The azure module will concentrate on retrieving different types of log data from Azure. +The module will contain the following filesets: + +`activitylogs` :: +Will retrieve azure activity logs. Control-plane events on Azure Resource Manager resources. Activity logs provide insight into the operations that were performed on resources in your subscription. + +`signinlogs` :: +Will retrieve azure Active Directory sign-in logs. The sign-ins report provides information about the usage of managed applications and user sign-in activities. + +`auditlogs` :: +Will retrieve azure Active Directory audit logs. The audit logs provide traceability through logs for all changes done by various features within Azure AD. Examples of audit logs include changes made to any resources within Azure AD like adding or removing users, apps, groups, roles and policies. + + +[float] +=== Module configuration + +``` +- module: azure + activitylogs: + enabled: true + var: + namespace: "obseventhubs.servicebus.windows.net:9093" + eventhub: ["insights-operational-logs"] + consumer_group: "$Default" + connection_string: "" + auditlogs: + enabled: true + var: + namespace: "" + eventhub: ["insights-logs-auditlogs"] + consumer_group: "$Default" + connection_string: "" + + signinlogs: + enabled: true + var: + namespace: "" + eventhub: ["insights-logs-signinlogs"] + consumer_group: "$Default" + connection_string: "" + +``` + + +A side by side kafka/event hubs notation, we will follow Azure notations in this case. + + +`namespace` :: +_string_ +An Event Hubs namespace provides a unique scoping container, referenced by its fully qualified domain name, in which users can create one or more event hubs or Kafka topics. + +`eventhub` :: + _[]string_ +Or kafka topic, is a fully managed, real-time data ingestion service. +Default value `insights-operational-logs` + +`consumer_group` :: +_string_ + The publish/subscribe mechanism of Event Hubs is enabled through consumer groups. A consumer group is a view (state, position, or offset) of an entire event hub. Consumer groups enable multiple consuming applications to each have a separate view of the event stream, and to read the stream independently at their own pace and with their own offsets. +Default value: `$Default` + +`connection_string` :: +_string_ +The connection string required to communicate with Event Hubs, steps here https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string. + + + include::../include/what-happens.asciidoc[] [float] @@ -30,28 +97,9 @@ This module comes with a sample dashboard. For example: TODO: include an image of a sample dashboard. If you do not include a dashboard, remove this section and set `:has-dashboards: false` at the top of this file. -include::../include/configuring-intro.asciidoc[] - -TODO: provide an example configuration - -:fileset_ex: {fileset} - -include::../include/config-option-intro.asciidoc[] - -TODO: document the variables from each fileset. If you're describing a variable -that's common to other modules, you can reuse shared descriptions by including -the relevant file. For example: - -[float] -==== `{fileset}` log fileset settings - -include::../include/var-paths.asciidoc[] -:has-dashboards!: -:fileset_ex!: -:modulename!: [float] diff --git a/filebeat/input/kafka/azure_log_patterns.go b/filebeat/input/kafka/azure_log_patterns.go index 06d283def84..fc651a0af34 100644 --- a/filebeat/input/kafka/azure_log_patterns.go +++ b/filebeat/input/kafka/azure_log_patterns.go @@ -20,11 +20,29 @@ package kafka import "time" const ( + // ActivityLogs variable used to identify the activitylogs azure metricset ActivityLogs = "ActivityLogs" - AuditLogs = "AuditLogs" - SigninLogs = "SigninLogs" + // AuditLogs variable used to identify the auditlogs azure metricset + AuditLogs = "AuditLogs" + // SigninLogs variable used to identify the signinlogs azure metricset + SigninLogs = "SigninLogs" ) +// AzureActivityLogs structure matches the eventhub message carrying the azure activity logs +type AzureActivityLogs struct { + Records []ActivityLog `json:"records"` +} + +// AzureAuditLogs structure matches the eventhub message carrying the azure audit logs +type AzureAuditLogs struct { + Records []AuditLog `json:"records"` +} + +// AzureSigninLogs structure matches the eventhub message carrying the azure signin logs +type AzureSigninLogs struct { + Records []SigninLog `json:"records"` +} + // ActivityLogs structure matches the azure activity log format type ActivityLog struct { Time time.Time `json:"time"` @@ -75,18 +93,7 @@ type ActivityLog struct { } `json:"properties,omitempty"` } -type AzureActivityLogs struct { - Records []ActivityLog `json:"records"` -} - -type AzureAuditLogs struct { - Records []AuditLog `json:"records"` -} - -type AzureSigninLogs struct { - Records []SigninLog `json:"records"` -} - +// AuditLog structure matches the azure audit log format type AuditLog struct { Category string `json:"category"` CorrelationID string `json:"correlationId"` @@ -138,6 +145,7 @@ type AuditLog struct { Identity string `json:"identity"` } +// SigninLog structure matches the azure audit log format type SigninLog struct { Level int `json:"Level"` CallerIPAddress string `json:"callerIpAddress"` diff --git a/filebeat/input/kafka/input.go b/filebeat/input/kafka/input.go index f16ee18da70..af64b94bf2c 100644 --- a/filebeat/input/kafka/input.go +++ b/filebeat/input/kafka/input.go @@ -109,6 +109,7 @@ func (input *kafkaInput) runConsumerGroup( handler := &groupHandler{ version: input.config.Version, outlet: input.outlet, + // logType will be assigned the configuration option AzureLogs, if the metricset using this input is an azure metricset then we can filter for the azure log types dedicated to the metricset (audit, activity, signin) logType: input.config.AzureLogs, } @@ -338,8 +339,6 @@ func (h *groupHandler) parseMultipleMessages(bMessage []byte) []string { case AuditLogs: // if the fileset is audit logs a filtering of the messages should be done as the eventhub can return different types of messages var obj AzureAuditLogs - test := string(bMessage) - _ = test err := json.Unmarshal(bMessage, &obj) if err != nil { return nil diff --git a/x-pack/filebeat/module/azure/_meta/docs.asciidoc b/x-pack/filebeat/module/azure/_meta/docs.asciidoc index cd17bfc2358..d352cdfb4c0 100644 --- a/x-pack/filebeat/module/azure/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/azure/_meta/docs.asciidoc @@ -7,6 +7,73 @@ beta[] This is the azure module. +The azure module will concentrate on retrieving different types of log data from Azure. +The module will contain the following filesets: + +`activitylogs` :: +Will retrieve azure activity logs. Control-plane events on Azure Resource Manager resources. Activity logs provide insight into the operations that were performed on resources in your subscription. + +`signinlogs` :: +Will retrieve azure Active Directory sign-in logs. The sign-ins report provides information about the usage of managed applications and user sign-in activities. + +`auditlogs` :: +Will retrieve azure Active Directory audit logs. The audit logs provide traceability through logs for all changes done by various features within Azure AD. Examples of audit logs include changes made to any resources within Azure AD like adding or removing users, apps, groups, roles and policies. + + +[float] +=== Module configuration + +``` +- module: azure + activitylogs: + enabled: true + var: + namespace: "obseventhubs.servicebus.windows.net:9093" + eventhub: ["insights-operational-logs"] + consumer_group: "$Default" + connection_string: "" + auditlogs: + enabled: true + var: + namespace: "" + eventhub: ["insights-logs-auditlogs"] + consumer_group: "$Default" + connection_string: "" + + signinlogs: + enabled: true + var: + namespace: "" + eventhub: ["insights-logs-signinlogs"] + consumer_group: "$Default" + connection_string: "" + +``` + + +A side by side kafka/event hubs notation, we will follow Azure notations in this case. + + +`namespace` :: +_string_ +An Event Hubs namespace provides a unique scoping container, referenced by its fully qualified domain name, in which users can create one or more event hubs or Kafka topics. + +`eventhub` :: + _[]string_ +Or kafka topic, is a fully managed, real-time data ingestion service. +Default value `insights-operational-logs` + +`consumer_group` :: +_string_ + The publish/subscribe mechanism of Event Hubs is enabled through consumer groups. A consumer group is a view (state, position, or offset) of an entire event hub. Consumer groups enable multiple consuming applications to each have a separate view of the event stream, and to read the stream independently at their own pace and with their own offsets. +Default value: `$Default` + +`connection_string` :: +_string_ +The connection string required to communicate with Event Hubs, steps here https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string. + + + include::../include/what-happens.asciidoc[] [float] @@ -25,25 +92,6 @@ This module comes with a sample dashboard. For example: TODO: include an image of a sample dashboard. If you do not include a dashboard, remove this section and set `:has-dashboards: false` at the top of this file. -include::../include/configuring-intro.asciidoc[] - -TODO: provide an example configuration - -:fileset_ex: {fileset} - -include::../include/config-option-intro.asciidoc[] - -TODO: document the variables from each fileset. If you're describing a variable -that's common to other modules, you can reuse shared descriptions by including -the relevant file. For example: - -[float] -==== `{fileset}` log fileset settings - -include::../include/var-paths.asciidoc[] -:has-dashboards!: -:fileset_ex!: -:modulename!: diff --git a/x-pack/filebeat/module/azure/activitylogs/test/activity_log.json b/x-pack/filebeat/module/azure/activitylogs/test/activity_log.json new file mode 100644 index 00000000000..2e4b6674c2c --- /dev/null +++ b/x-pack/filebeat/module/azure/activitylogs/test/activity_log.json @@ -0,0 +1,95 @@ +{ + "records": [ + { + "time": "2019-10-04T10:38:37.5151977Z", + "resourceId": "/SUBSCRIPTIONS/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/RESOURCEGROUPS/SA-HEMANT/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/AZURELSEVENTS/AUTHORIZATIONRULES/ROOTMANAGESHAREDACCESSKEY", + "operationName": "MICROSOFT.EVENTHUB/NAMESPACES/AUTHORIZATIONRULES/LISTKEYS/ACTION", + "category": "Action", + "resultType": "Start", + "resultSignature": "Started.", + "durationMs": 0, + "callerIpAddress": "51.191.161.11", + "correlationId": "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "identity": { + "authorization": { + "scope": "/subscriptions/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/resourceGroups/sa-hemant/providers/Microsoft.EventHub/namespaces/azurelsevents/authorizationRules/RootManageSharedAccessKey", + "action": "Microsoft.EventHub/namespaces/authorizationRules/listKeys/action", + "evidence": { + "role": "Azure EventGrid Service BuiltIn Role", + "roleAssignmentScope": "/subscriptions/2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "roleAssignmentId": "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "roleDefinitionId": "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "principalId": "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "principalType": "ServicePrincipal" + } + }, + "claims": { + "aud": "https://management.core.windows.net/", + "iss": "https://sts.windows.net/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/", + "iat": "1570185217", + "nbf": "1570185217", + "exp": "1570214317", + "aio": "2a7e2503-d7e2", + "appid": "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "appidacr": "2", + "http://schemas.microsoft.com/identity/claims/identityprovider": "https://sts.windows.net/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/", + "http://schemas.microsoft.com/identity/claims/objectidentifier": "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "http://schemas.xmlsoap.org/ws/2005/05/identity/claims/nameidentifier": "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "http://schemas.microsoft.com/identity/claims/tenantid": "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "uti": "2a7e2503", + "ver": "1.0" + } + }, + "level": "Information", + "location": "global" + }, + { + "time": "2019-10-04T10:38:37.5251981Z", + "resourceId": "/SUBSCRIPTIONS/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/RESOURCEGROUPS/SA-HEMANT/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/AZURELSEVENTS/AUTHORIZATIONRULES/ROOTMANAGESHAREDACCESSKEY", + "operationName": "MICROSOFT.EVENTHUB/NAMESPACES/AUTHORIZATIONRULES/LISTKEYS/ACTION", + "category": "Action", + "resultType": "Failure", + "resultSignature": "Failed.NotFound", + "durationMs": 11, + "callerIpAddress": "51.191.161.11", + "correlationId": "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "identity": { + "authorization": { + "scope": "/subscriptions/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/resourceGroups/sa-hemant/providers/Microsoft.EventHub/namespaces/azurelsevents/authorizationRules/RootManageSharedAccessKey", + "action": "Microsoft.EventHub/namespaces/authorizationRules/listKeys/action", + "evidence": { + "role": "Azure EventGrid Service BuiltIn Role", + "roleAssignmentScope": "/subscriptions/2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "roleAssignmentId": "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "roleDefinitionId": "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "principalId": "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "principalType": "ServicePrincipal" + } + }, + "claims": { + "aud": "https://management.core.windows.net/", + "iss": "https://sts.windows.net/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/", + "iat": "1570185217", + "nbf": "1570185217", + "exp": "1570214317", + "aio": "2a7e2503", + "appid": "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "appidacr": "2", + "http://schemas.microsoft.com/identity/claims/identityprovider": "https://sts.windows.net/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/", + "http://schemas.microsoft.com/identity/claims/objectidentifier": "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "http://schemas.xmlsoap.org/ws/2005/05/identity/claims/nameidentifier": "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "http://schemas.microsoft.com/identity/claims/tenantid": "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "uti": "2a7e2503", + "ver": "1.0" + } + }, + "level": "Error", + "location": "global", + "properties": { + "statusCode": "NotFound", + "serviceRequestId": null, + "statusMessage": "{\"error\":{\"code\":\"ParentResourceNotFound\",\"message\":\"Can not perform requested operation on nested resource. Parent resource 'azurelsevents' not found.\"}}" + } + } + ] +} diff --git a/x-pack/filebeat/module/azure/activitylogs/test/activity_log_expected.json b/x-pack/filebeat/module/azure/activitylogs/test/activity_log_expected.json index e69de29bb2d..766b5dda06a 100644 --- a/x-pack/filebeat/module/azure/activitylogs/test/activity_log_expected.json +++ b/x-pack/filebeat/module/azure/activitylogs/test/activity_log_expected.json @@ -0,0 +1,103 @@ +{ + "_index" : "filebeat-8.0.0-2019.09.26", + "_type" : "_doc", + "_id" : "2a7e2503-d7e2", + "_score" : null, + "_source" : { + "agent" : { + "hostname" : "DESKTOP-RFOOE09", + "id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "type" : "filebeat", + "ephemeral_id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "version" : "8.0.0" + }, + "ecs" : { + "version" : "1.1.0" + }, + "identity" : { + "authorization" : { + "evidence" : { + "roleAssignmentScope" : "/subscriptions/2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "roleDefinitionId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "role" : "Azure EventGrid Service BuiltIn Role", + "roleAssignmentId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "principalId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "principalType" : "ServicePrincipal" + }, + "scope" : "/subscriptions/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/resourceGroups/sa-hemant/providers/Microsoft.EventHub/namespaces/azurelsevents/authorizationRules/RootManageSharedAccessKey", + "action" : "Microsoft.EventHub/namespaces/authorizationRules/listKeys/action" + }, + "claims" : { + "http://schemas.xmlsoap.org/ws/2005/05/identity/claims/nameidentifier" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "ver" : "1.0", + "http://schemas.microsoft.com/identity/claims/identityprovider" : "https://sts.windows.net/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/", + "aio" : "2a7e2503-d7e2-40", + "iss" : "https://sts.windows.net/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/", + "uti" : "2a7e2503-d7e2", + "aud" : "https://management.core.windows.net/", + "nbf" : "1569500393", + "appidacr" : "2", + "http://schemas.microsoft.com/identity/claims/tenantid" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "appid" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "http://schemas.microsoft.com/identity/claims/objectidentifier" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "exp" : "1569529493", + "iat" : "1569500393" + } + }, + "host" : { + "hostname" : "DESKTOP-RFOOE09", + "os" : { + "build" : "17134.1006", + "kernel" : "10.0.17134.1006 (WinBuild.160101.0800)", + "name" : "Windows 10 Pro", + "family" : "windows", + "version" : "10.0", + "platform" : "windows" + }, + "name" : "DESKTOP-RFOOE09", + "id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "architecture" : "x86_64" + }, + "correlationId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "event" : { + "module" : "azure", + "dataset" : "azure.activitylogs" + }, + "resultType" : "Failure", + "callerIpAddress" : "11.71.131.141", + "resourceID" : "/SUBSCRIPTIONS/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/RESOURCEGROUPS/SA-HEMANT/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/AZURELSEVENTS/AUTHORIZATIONRULES/ROOTMANAGESHAREDACCESSKEY", + "level" : "Error", + "operationName" : "MICROSOFT.EVENTHUB/NAMESPACES/AUTHORIZATIONRULES/LISTKEYS/ACTION", + "fileset" : { + "name" : "activitylogs" + }, + "resultSignature" : "Failed.NotFound", + "input" : { + "type" : "azure" + }, + "@timestamp" : "2019-09-26T12:24:53.623Z", + "service" : { + "type" : "azure" + }, + "kafka" : { + "headers" : [ + "ProfileName: �\fname=default" + ], + "partition" : 2, + "offset" : 12887, + "topic" : "insights-operational-logs", + "key" : "" + }, + "location" : "global", + "category" : "Action", + "durationMs" : 11, + "properties" : { + "serviceRequestId" : null, + "statusMessage" : """{"error":{"code":"ParentResourceNotFound","message":"Can not perform requested operation on nested resource. Parent resource 'azurelsevents' not found."}}""", + "statusCode" : "NotFound" + } + }, + "sort" : [ + 1569500693623 + ] +} diff --git a/x-pack/filebeat/module/azure/auditlogs/_meta/fields.yml b/x-pack/filebeat/module/azure/auditlogs/_meta/fields.yml index 30ebf2a29fd..4f0fa2df629 100644 --- a/x-pack/filebeat/module/azure/auditlogs/_meta/fields.yml +++ b/x-pack/filebeat/module/azure/auditlogs/_meta/fields.yml @@ -3,3 +3,173 @@ description: > Fields for azure audit logs. fields: + - name: resourceID + type: keyword + description: > + Resource ID + - name: operationName + type: keyword + description: > + The operation name + - name: operationVersion + type: keyword + description: > + The operation version + - name: category + type: keyword + description: > + Log category + - name: tenantId + type: keyword + description: > + Tenant ID + - name: resultSignature + type: keyword + description: > + Result signature + - name: durationMs + type: keyword + description: > + Activity duration + - name: correlationId + type: keyword + description: > + Correlation ID + - name: level + type: keyword + description: > + Log level + - name: properties + type: group + description: > + The audit log properties + fields: + - name: result + type: keyword + description: > + Log result + - name: activityDisplayName + type: keyword + description: > + Activity display name + - name: resultReason + type: keyword + description: > + Reason for the log result + - name: correlationId + type: keyword + description: > + Correlation ID + - name: loggedByService + type: keyword + description: > + Logged by service + - name: operationType + type: keyword + description: > + Operation type + - name: id + type: keyword + description: > + ID + - name: activityDateTime + type: keyword + description: > + Activity timestamp + - name: category + type: keyword + description: > + category + - name: targetResources + type: group + description: > + Target resources + fields: + - name: displayName + type: keyword + description: > + Display name + - name: id + type: keyword + description: > + ID + - name: type + type: keyword + description: > + Type + - name: ipAddress + type: keyword + description: > + ip Address + - name: userPrincipalName + type: keyword + description: > + User principal name + - name: modifiedProperties + type: group + description: > + Modified properties + fields: + - name: newValue + type: keyword + description: > + New value + - name: displayName + type: keyword + description: > + Display value + - name: oldValue + type: keyword + description: > + Old value + - name: initiatedBy + type: group + description: > + Information regarding the initiator + fields: + - name: app + type: group + description: > + App + fields: + - name: servicePrincipalName + type: keyword + description: > + Service principal name + - name: displayName + type: keyword + description: > + Display name + - name: appId + type: keyword + description: > + App ID + - name: servicePrincipalId + type: keyword + description: > + Service principal ID + - name: user + type: group + description: > + User + fields: + - name: userPrincipalName + type: keyword + description: > + User principal name + - name: displayName + type: keyword + description: > + Display name + - name: id + type: keyword + description: > + ID + - name: ipAddress + type: keyword + description: > + ip Address + + + + diff --git a/x-pack/filebeat/module/azure/auditlogs/test/audit_log.json b/x-pack/filebeat/module/azure/auditlogs/test/audit_log.json new file mode 100644 index 00000000000..7d0f57c6afb --- /dev/null +++ b/x-pack/filebeat/module/azure/auditlogs/test/audit_log.json @@ -0,0 +1,78 @@ +{ + "records": [ + { + "time":"2019-10-04T16:36:19.0359773Z", + "resourceId":"/tenants/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/providers/Microsoft.aadiam", + "operationName":"Add user", + "operationVersion":"1.0", + "category":"AuditLogs", + "tenantId":"2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "resultSignature":"None", + "durationMs":0, + "correlationId":"2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "level":"Informational", + "properties":{ + "id":"Directory_T2a7e2503-d7e2", + "category":"UserManagement", + "correlationId":"2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "result":"success", + "resultReason":"", + "activityDisplayName":"Add user", + "activityDateTime":"2019-10-04T16:36:19.0359773+00:00", + "loggedByService":"Core Directory", + "operationType":"Add", + "initiatedBy":{ + "user":{ + "id":"2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "displayName":null, + "userPrincipalName":"admin@elastic365.onmicrosoft.com", + "ipAddress":null + } + }, + "targetResources":[ + { + "id":"2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "displayName":null, + "type":"User", + "userPrincipalName":"test_testd@elastic.co", + "modifiedProperties":[ + { + "displayName":"AccountEnabled", + "oldValue":"[]", + "newValue":"[true]" + }, + { + "displayName":"PasswordPolicies", + "oldValue":"[]", + "newValue":"[\"None\"]" + }, + { + "displayName":"SourceAnchor", + "oldValue":"[]", + "newValue":"[\"2a7e2503\"]" + }, + { + "displayName":"UserPrincipalName", + "oldValue":"[]", + "newValue":"[\"test_testd@elastic.co\"]" + }, + { + "displayName":"UserType", + "oldValue":"[]", + "newValue":"[\"Member\"]" + }, + { + "displayName":"Included Updated Properties", + "oldValue":null, + "newValue":"\"AccountEnabled, PasswordPolicies, SourceAnchor, UserPrincipalName, UserType\"" + } + ] + } + ], + "additionalDetails":[ + + ] + } + } + ] +} diff --git a/x-pack/filebeat/module/azure/auditlogs/test/audit_log_expected.json b/x-pack/filebeat/module/azure/auditlogs/test/audit_log_expected.json new file mode 100644 index 00000000000..3d3a9d95e22 --- /dev/null +++ b/x-pack/filebeat/module/azure/auditlogs/test/audit_log_expected.json @@ -0,0 +1,107 @@ +{ + "_index" : "filebeat-8.0.0-2019.10.04", + "_type" : "_doc", + "_id" : "FXFblm0Bq15PGrhqyTlD", + "_score" : null, + "_source" : { + "agent" : { + "hostname" : "DESKTOP-RFOOE09", + "id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "type" : "filebeat", + "ephemeral_id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "version" : "8.0.0" + }, + "ecs" : { + "version" : "1.1.0" + }, + "host" : { + "hostname" : "DESKTOP-RFOOE09", + "os" : { + "build" : "18362.387", + "kernel" : "10.0.18362.387 (WinBuild.160101.0800)", + "name" : "Windows 10 Pro", + "family" : "windows", + "version" : "10.0", + "platform" : "windows" + }, + "name" : "DESKTOP-RFOOE09", + "id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "architecture" : "x86_64" + }, + "correlationId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "event" : { + "module" : "azure", + "dataset" : "azure.auditlogs" + }, + "resourceID" : "/tenants/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/providers/Microsoft.aadiam", + "level" : "Informational", + "specification" : "Device Registration Service", + "operationName" : "Update device", + "fileset" : { + "name" : "auditlogs" + }, + "resultSignature" : "None", + "input" : { + "type" : "kafka" + }, + "@timestamp" : "2019-10-04T10:05:32.944Z", + "operationVersion" : "1.0", + "service" : { + "type" : "azure" + }, + "kafka" : { + "headers" : [ ], + "partition" : 3, + "offset" : 2, + "topic" : "insights-logs-auditlogs", + "key" : "" + }, + "tenantId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "category" : "AuditLogs", + "durationMs" : 0, + "properties" : { + "result" : "success", + "activityDisplayName" : "Update device", + "resultReason" : "", + "correlationId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "loggedByService" : "Core Directory", + "operationType" : "Update", + "id" : "Directory_2a7e2503", + "activityDateTime" : "2019-10-04T10:05:32.9448766+00:00", + "additionalDetails" : [ ], + "targetResources" : [ + { + "displayName" : "win10odbc", + "modifiedProperties" : [ + { + "newValue" : "", + "displayName" : "Included Updated Properties", + "oldValue" : "" + } + ], + "id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "type" : "Device", + "userPrincipalName" : "" + } + ], + "category" : "Device", + "initiatedBy" : { + "app" : { + "servicePrincipalName" : null, + "displayName" : "Device Registration Service", + "appId" : null, + "servicePrincipalId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73" + }, + "user" : { + "displayName" : null, + "ipAddress" : null, + "id" : "", + "userPrincipalName" : "" + } + } + } + }, + "sort" : [ + 1570183532944 + ] +} diff --git a/x-pack/filebeat/module/azure/fields.go b/x-pack/filebeat/module/azure/fields.go index d722070c700..153c412a23e 100644 --- a/x-pack/filebeat/module/azure/fields.go +++ b/x-pack/filebeat/module/azure/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAzure returns asset data. // This is the base64 encoded gzipped contents of module/azure. func AssetAzure() string { - return "eJzEl91uIiEUx+99ihMvm9QH8GITU7OJyW63aXvfIBzHsyIQYGymT79hdD50mA93XJc7hfn/fwcOcHiEHWZzYF+pxQmAJy9xDtP893QCINBxS8aTVnP4NgGA41j4qUUqwycbQincPO96BMX2WMmF5jODc0isTs3pn4jmucyZFPd0IJ9JnbiyM6baqnxs33N92Gh7CmBxEoagPKuNvSSp05BA5clnZ51tND1Eob1vEThTWhFnElKHFlZL0BvwWwT9qdAWP5xOLUdYp3yHfnYhFEOuY3PJaO9mD40BBbpe/0buI93Hjo/jqB1mn9qK7mEfe2YMqeT0zfRhGhnfMy2hPeXMrSGx1G+1pS8WNFrjii3JQP9nLRCYlJozz9YSwWjRxGmf+zosHkLicIwO6mMdyHsNcx93nd1qiQvnKFF7VP6Na9MWRhVKe6JcGVBor1oisJIAXCdCHXuJG1IUPFZdLP+KWZT2sFoOAr4/5JXL/1/msbb2A+bRWFKcDJP3Zn0pjK+ifM/uvZ8qTh/zLgjbtlkf0QCS+CFSv/E7jvRRzoumdnlHamtRsuhp0eXc4/pUyV7mRbnF0KXSRxJhhO1rrgkN0TJaJiXalVkIYdFd3hFj4s2FYWWARaRrIef1TGOnjAv5WCO1TLPEA8rb2f1oyBVG2qDNF/yZ7W+4pL8K2dymI5PeKFHMVxX4DbxP6eSiytWZFiL3hM1Ks6PKHFBhXlNd9gTyUjLGcyRUTs3TZ0yaxBSrXegx0Tb+nPjL7Xeh2HxQpYL8zV9TuWrvU6q8WyhRpO4M8ScAAP//XRKL7w==" + return "eJzsW81u2zgQvucpBjkWaB8ghwVce3dhoGmK/PRaMNJE4YYmtSTlwH36BSnrxxYp0TFpZ4Ho1FTKfB/JGc7HGeYzvODmCsjvSuIFgKaa4RVc2p8vLwByVJmkpaaCX8EfFwBQfwvXIq+Y+ZUniixXV/bVZ+BkhZ058+hNiVdQSFGV2/9x2Nw1s2Mq03RN9YaJQrUvXVa9luvnL2sfnoTcDmC2NQzG8pfet/tM+mxojlxTvdl56WMzwcg8988IGeGC04wwqBRKWC5APIF+RhCvHGXzgxKVzBAeq+wF9Zc9Qy7KfdoZI3SlvnwafNBQF4//YKYdr+sXv+qvXnDzKmQ+/tmvFSlLyovt71x+unR8PzEt5plbzt4hkUo/C0l/E2PDOy7XkgTifxc5AmFMZESTR4ZQinxIxz/3fbK4No6TofOjKa6BfA/hPMW7z10KhjOlaMFXyPVdJkrfMLqh+B3lwAGZ51YwBNIyADVKoU97gU+UU4OxHOOSinPewsNyEUT49CQPXP6zzGNv7QPmsZSUZ7Qk7NRcfzTAB7G835w6njqe2oXdMPSF2RSjACbuTaSf8Ue29KOQZ0PbbY4UUiIjzt1iDHkCdd6Z3feLNsRQVUw7HOEI2FtrEwZG29ESxlAuy1meS1T7OeKY8VrDsCyBOEz3hmz1zCBSjhtyrZE808xwjSwe3LeBuQZIlCjtgn8nq4hLetOYtTAjnnRHC050p8AjYG/dSTktd3uaGbmmOFSaIyozQGEeoi4nBvKj5ej2EaOchrvPMW7isthFocZCSPdx4o3ht2dxeKCqcqqjn6as1YOOUufYAhJFpjnFienobD/5iVJFdbJdAmuH+XQe900UbqsNpEZOHMrxmOFai+Pp9FybYF7Vq3AdMae2pYrG+DsTMLEzqygcJodZxon4xiJMu3/57U8VWWq/S6NbzZR47O8XyhZUlYxsHDtcJC6dN9ZIww1vOCu3SFQqTV/btrlIP6Ndw4mpGouUSKRGogZ29EZRYP51c4dyTZ31oTjOU2AOjxtQHphBhvKcSiOQ6VTs6OmTJlqXkbVoA4hovKfJo0fTFSpNVsOS30SujkTEa73N20QWqBtJ5SojHl1jvbcQrRR8a3k1H93yILCMElhCWYztejDlwpHJeCpN7RL6i0sRSXhLWO1MeMoMCbgALWEMrKFUKZRtIew0jvOgUHZ1v2n/WYmcPlHMPUfWfY4R2gfXW0S/Bmqe0OYBx9efhFUnrnB+x1dYj8KG7xyJKDa7SBhNwfIzTOMNyz382sDmVFOijYJJkR6W/EnIVa0WJBZE5pQXVuRtgYV8Y8YgpS9UogXTzAsRGjxbtRayTUEaD9jK0pBdC95TVAWxJGV56hbRrCxDmkP7C39qmsNln9AZJpumjqcHP0ZoQIUmfUgzraECAP53oeSVuonIBcTQlOZMxMwhP0P+0UY+LTjlH9X5j+p8JOyP6vwO+kd1/mjE8ep8wrsMg2LGO7nLMHon9E24S5fFM7Wo07Rb6jx3XL/lDCXiTKI57aatEM9rEMiJRlsl9tIxejZ5u8eq1qBWz/tpq4QI/VhTMyHo+5RSzcpDfW/ci+076cZoZ7hPtD3k9P3Isgzzz7EjQYx9w50mYOcePkWuZ2X5oDBVhFgIM/HG5YZGumDlub2bTNgsy1CpO010lWhq5h0YEIsGyg3XqnFJC8oJu8V/K1Tue8cxupFbGJA1zpgvU7XkGqVtEyZy5aUCOoLRanbxgnypVIUyXVjdGxCgFmU8rnp00vWNd+iMto5LKYyHUV6YHL3k15QxqtA4fCL3/tEi2nwNlMNqDLQVy1S9LFATui/PI/G6peoFcjdAn4O9yjorComFUR4JydiDAxA/1IDWopKUF+a8uEx1f6Qjllswq04daH1uZrdM5OmWj3La36/OJM+tbckmKMF69vQInSBvcgr66ysphZyL/AQ93j8NFGQurLYEgWua4UTYn2uqanLefkPMaxQWaaqxsC2b8eJuozSu0tO6aQBB+REbdo9SvE51PqKw+uoBmqgJwNn9KRvWRXaJRZmeuQ+lvzGdIPp9SaGvuiuu5eZG3mLhXq/IlOY1IAgJ0g/Z0CtQzIWQOeVEp79u8ncIWmhvjxFNdeXd4yFNu+fbFGoXobw4B78B7MV/AQAA//+Kx76S" } diff --git a/x-pack/filebeat/module/azure/signinlogs/_meta/fields.yml b/x-pack/filebeat/module/azure/signinlogs/_meta/fields.yml index 7b5d583554a..22f8f48040a 100644 --- a/x-pack/filebeat/module/azure/signinlogs/_meta/fields.yml +++ b/x-pack/filebeat/module/azure/signinlogs/_meta/fields.yml @@ -3,3 +3,201 @@ description: > Fields for azure audit logs. fields: + - name: resourceID + type: keyword + description: > + Resource ID + - name: operationName + type: keyword + description: > + The operation name + - name: operationVersion + type: keyword + description: > + The operation version + - name: category + type: keyword + description: > + Log category + - name: tenantId + type: keyword + description: > + Tenant ID + - name: resultSignature + type: keyword + description: > + Result signature + - name: durationMs + type: keyword + description: > + Activity duration + - name: correlationId + type: keyword + description: > + Correlation ID + - name: level + type: keyword + description: > + Log level + - name: resultType + type: keyword + description: > + Result type + - name: callerIpAddress + type: keyword + description: > + Caller Ip address + - name: identity + type: keyword + description: > + Identity + - name: location + type: keyword + description: > + Location + - name: properties + type: group + description: > + The signin log properties + fields: + - name: id + type: keyword + description: > + ID + - name: createdDateTime + type: keyword + description: > + Created date time + - name: userDisplayName + type: keyword + description: > + User display name + - name: correlationId + type: keyword + description: > + Correlation ID + - name: userPrincipalName + type: keyword + description: > + User principal name + - name: userId + type: keyword + description: > + User ID + - name: appId + type: keyword + description: > + App ID + - name: appDisplayName + type: keyword + description: > + App display name + - name: ipAddress + type: keyword + description: > + Ip address + - name: clientAppUsed + type: keyword + description: > + Client app used + - name: conditionalAccessStatus + type: keyword + description: > + Conditional access status + - name: originalRequestId + type: keyword + description: > + Original request ID + - name: isInteractive + type: keyword + description: > + Is interactive + - name: tokenIssuerName + type: keyword + description: > + Token issuer name + - name: tokenIssuerType + type: keyword + description: > + Token issuer type + - name: processingTimeInMilliseconds + type: keyword + description: > + Processing time in milliseconds + - name: riskDetail + type: keyword + description: > + Risk detail + - name: riskLevelAggregated + type: keyword + description: > + Risk level aggregated + - name: riskLevelDuringSignIn + type: keyword + description: > + Risk level during signIn + - name: riskState + type: keyword + description: > + Risk state + - name: resourceDisplayName + type: keyword + description: > + Resource display name + - name: status + type: group + description: > + Status + fields: + - name: errorCode + type: keyword + description: > + Error code + - name: deviceDetail + type: group + description: > + Status + fields: + - name: deviceId + type: keyword + description: > + Device ID + - name: operatingSystem + type: keyword + description: > + Operating system + - name: browser + type: keyword + description: > + Browser + - name: location + type: group + description: > + Status + fields: + - name: city + type: keyword + description: > + City + - name: state + type: keyword + description: > + State + - name: countryOrRegion + type: keyword + description: > + Country or region + - name: geoCoordinates + type: group + description: > + GeoCoordinates + fields: + - name: latitude + type: keyword + description: > + Latitude + - name: longitude + type: keyword + description: > + Longitude + diff --git a/x-pack/filebeat/module/azure/signinlogs/test/signin_log.json b/x-pack/filebeat/module/azure/signinlogs/test/signin_log.json new file mode 100644 index 00000000000..f2c07d16193 --- /dev/null +++ b/x-pack/filebeat/module/azure/signinlogs/test/signin_log.json @@ -0,0 +1,155 @@ +{ + "time":"2019-10-04T10:47:34.5949655Z", + "resourceId":"/tenants/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/providers/Microsoft.aadiam", + "operationName":"Sign-in activity", + "operationVersion":"1.0", + "category":"SignInLogs", + "tenantId":"2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "resultType":"0", + "resultSignature":"None", + "durationMs":0, + "callerIpAddress":"81.191.171.1", + "correlationId":"2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "identity":"Test Test", + "Level":4, + "location":"GB", + "properties":{ + "SignInBondData":{ + "LocationDetails":{ + "Latitude":51.11111, + "Longitude":-2.1111, + "IPChain":null + }, + "MfaDetails":null, + "ConditionalAccessDetails":{ + "MultiCAEvaluationLog":"0|" + }, + "RamDetails":null, + "DeviceDetails":{ + "DeviceDisplayName":null, + "DeviceId":"", + "IsCompliant":null, + "IsManaged":null, + "DeviceTrustType":null, + "UserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36", + "DevicePlatform":"MacOs", + "BrowserId":"2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "BrowserType":"Chrome" + }, + "DisplayDetails":{ + "UserName":"Test Test", + "ApplicationDisplayName":"Microsoft Office Web Apps Service", + "ProxyRestrictionTargetTenantName":null, + "ResourceDisplayName":"Windows Azure Active Directory", + "AttemptedUsername":null + }, + "ProtocolDetails":{ + "ResponseTime":131, + "IsInteractive":null, + "AuthenticationMethodsUsed":null, + "NetworkLocation":null, + "DomainHintPresent":null, + "LoginHintPresent":null, + "Protocol":null + }, + "PassThroughAuthenticationDetails":null, + "IssuerDetails":null, + "SourceAlpEvents":null + }, + "id":"2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "createdDateTime":"2019-10-04T10:47:34.5949655+00:00", + "userDisplayName":"Test Test", + "userPrincipalName":"test_test@elastic.co", + "userId":"2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "appId":"2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "appDisplayName":"Microsoft Office Web Apps Service", + "ipAddress":"81.191.171.1", + "status":{ + "errorCode":0 + }, + "clientAppUsed":"Browser", + "deviceDetail":{ + "deviceId":"", + "operatingSystem":"MacOs", + "browser":"Chrome 73.0.3683" + }, + "location":{ + "city":"City1", + "state":"State1", + "countryOrRegion":"GB", + "geoCoordinates":{ + "latitude":51.111111111111, + "longitude":-2.1111111111111111111 + } + }, + "correlationId":"2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "conditionalAccessStatus":"notApplied", + "appliedConditionalAccessPolicies":[ + { + "id":"2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "displayName":"Require MFA for Azure", + "enforcedGrantControls":[ + "Mfa" + ], + "enforcedSessionControls":[ + + ], + "result":"notApplied", + "conditionsSatisfied":1, + "conditionsNotSatisfied":2 + }, + { + "id":"2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "displayName":"Disable Persistent Browser Sessions", + "enforcedGrantControls":[ + + ], + "enforcedSessionControls":[ + + ], + "result":"notEnabled", + "conditionsSatisfied":0, + "conditionsNotSatisfied":0 + }, + { + "id":"2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "displayName":"12-hour Session timeouts", + "enforcedGrantControls":[ + + ], + "enforcedSessionControls":[ + + ], + "result":"notEnabled", + "conditionsSatisfied":0, + "conditionsNotSatisfied":0 + } + ], + "originalRequestId":"2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "isInteractive":false, + "tokenIssuerName":"", + "tokenIssuerType":"AzureAD", + "authenticationProcessingDetails":[ + + ], + "networkLocationDetails":[ + + ], + "processingTimeInMilliseconds":131, + "riskDetail":"none", + "riskLevelAggregated":"none", + "riskLevelDuringSignIn":"none", + "riskState":"none", + "riskEventTypes":[ + + ], + "resourceDisplayName":"Windows Azure Active Directory", + "resourceId":"2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "authenticationDetails":[ + + ], + "authenticationRequirementPolicies":[ + + ] + } +} diff --git a/x-pack/filebeat/module/azure/activitylogs/test/activityLog.json b/x-pack/filebeat/module/azure/signinlogs/test/signin_log_expected.json similarity index 100% rename from x-pack/filebeat/module/azure/activitylogs/test/activityLog.json rename to x-pack/filebeat/module/azure/signinlogs/test/signin_log_expected.json From 766026e0f4d0e34a85f82ff975b5d26dfcd91081 Mon Sep 17 00:00:00 2001 From: Mariana Date: Tue, 8 Oct 2019 10:21:46 +0200 Subject: [PATCH 12/21] Work on kafka logtype --- filebeat/input/kafka/azure_log_patterns.go | 9 --- filebeat/input/kafka/config.go | 9 ++- filebeat/input/kafka/input.go | 10 +-- libbeat/common/kafka/logtype.go | 65 +++++++++++++++++++ .../activitylogs/config/activitylogs.yml | 2 +- .../azure/auditlogs/config/auditlogs.yml | 2 +- .../azure/signinlogs/config/signinlogs.yml | 2 +- 7 files changed, 81 insertions(+), 18 deletions(-) create mode 100644 libbeat/common/kafka/logtype.go diff --git a/filebeat/input/kafka/azure_log_patterns.go b/filebeat/input/kafka/azure_log_patterns.go index fc651a0af34..7998dd17f57 100644 --- a/filebeat/input/kafka/azure_log_patterns.go +++ b/filebeat/input/kafka/azure_log_patterns.go @@ -19,15 +19,6 @@ package kafka import "time" -const ( - // ActivityLogs variable used to identify the activitylogs azure metricset - ActivityLogs = "ActivityLogs" - // AuditLogs variable used to identify the auditlogs azure metricset - AuditLogs = "AuditLogs" - // SigninLogs variable used to identify the signinlogs azure metricset - SigninLogs = "SigninLogs" -) - // AzureActivityLogs structure matches the eventhub message carrying the azure activity logs type AzureActivityLogs struct { Records []ActivityLog `json:"records"` diff --git a/filebeat/input/kafka/config.go b/filebeat/input/kafka/config.go index 8e9a34cfd07..039d6c43a8c 100644 --- a/filebeat/input/kafka/config.go +++ b/filebeat/input/kafka/config.go @@ -50,7 +50,7 @@ type kafkaInputConfig struct { TLS *tlscommon.Config `config:"ssl"` Username string `config:"username"` Password string `config:"password"` - AzureLogs string `config:"azure_logs"` + LogType kafka.LogType `config:"log_types"` } type kafkaFetch struct { @@ -138,6 +138,13 @@ func (c *kafkaInputConfig) Validate() error { return err } + // if any log types are specified by the metricset config then a validation of the type will be done, else, no extra processing will happen to the messages + if c.LogType != "" { + if err := c.LogType.Validate(); err != nil { + return err + } + } + if c.Username != "" && c.Password == "" { return fmt.Errorf("password must be set when username is configured") } diff --git a/filebeat/input/kafka/input.go b/filebeat/input/kafka/input.go index af64b94bf2c..23c12a74b3d 100644 --- a/filebeat/input/kafka/input.go +++ b/filebeat/input/kafka/input.go @@ -110,7 +110,7 @@ func (input *kafkaInput) runConsumerGroup( version: input.config.Version, outlet: input.outlet, // logType will be assigned the configuration option AzureLogs, if the metricset using this input is an azure metricset then we can filter for the azure log types dedicated to the metricset (audit, activity, signin) - logType: input.config.AzureLogs, + logType: input.config.LogType, } input.saramaWaitGroup.Add(1) @@ -238,7 +238,7 @@ type groupHandler struct { version kafka.Version session sarama.ConsumerGroupSession outlet channel.Outleter - logType string + logType kafka.LogType } // The metadata attached to incoming events so they can be ACKed once they've @@ -336,7 +336,7 @@ func (h *groupHandler) parseMultipleMessages(bMessage []byte) []string { case "": default: return []string{string(bMessage)} - case AuditLogs: + case kafka.AuditLogs: // if the fileset is audit logs a filtering of the messages should be done as the eventhub can return different types of messages var obj AzureAuditLogs err := json.Unmarshal(bMessage, &obj) @@ -350,7 +350,7 @@ func (h *groupHandler) parseMultipleMessages(bMessage []byte) []string { } } return messages - case ActivityLogs: + case kafka.ActivityLogs: // if the fileset is activity logs a filtering of the messages should be done as the eventhub can return different types of messages var obj AzureActivityLogs err := json.Unmarshal(bMessage, &obj) @@ -364,7 +364,7 @@ func (h *groupHandler) parseMultipleMessages(bMessage []byte) []string { } } return messages - case SigninLogs: + case kafka.SigninLogs: // if the fileset is signin logs a filtering of the messages should be done as the eventhub can return different types of messages var obj AzureSigninLogs err := json.Unmarshal(bMessage, &obj) diff --git a/libbeat/common/kafka/logtype.go b/libbeat/common/kafka/logtype.go new file mode 100644 index 00000000000..b45cf0fa95f --- /dev/null +++ b/libbeat/common/kafka/logtype.go @@ -0,0 +1,65 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package kafka + +import ( + "fmt" +) + +// LogType is the type of the messages coming in +type LogType string + +var ( + // ActivityLogs variable used to identify the activitylogs azure metricset + ActivityLogs LogType = "ActivityLogs" + // AuditLogs variable used to identify the auditlogs azure metricset + AuditLogs LogType = "AuditLogs" + // SigninLogs variable used to identify the signinlogs azure metricset + SigninLogs LogType = "SigninLogs" + + LogTypes = map[string]LogType{ + "AuditLogs": AuditLogs, + "ActivityLogs": ActivityLogs, + "SigninLogs": SigninLogs, + } +) + +// Validate that a log type is among the possible options +func (lt *LogType) Validate() error { + if _, ok := LogTypes[string(*lt)]; !ok { + return fmt.Errorf("unknown/unsupported kafka vesion '%v'", *lt) + } + + return nil +} + +// Unpack the log type version +func (lt *LogType) Unpack(s string) error { + tmp := LogType(s) + if err := tmp.Validate(); err != nil { + return err + } + *lt = tmp + return nil +} + +// Get the log type +func (lt LogType) Get() (LogType, bool) { + kv, ok := LogTypes[string(lt)] + return kv, ok +} diff --git a/x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml b/x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml index 2a95c87cb14..0018b039f62 100644 --- a/x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml +++ b/x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml @@ -2,7 +2,7 @@ type: kafka hosts: {{ .namespace }} topics: {{ .eventhub }} group_id: {{ .consumer_group }} -azure_logs: "ActivityLogs" +log_types: "ActivityLogs" username: "$ConnectionString" password: {{ .connection_string }} diff --git a/x-pack/filebeat/module/azure/auditlogs/config/auditlogs.yml b/x-pack/filebeat/module/azure/auditlogs/config/auditlogs.yml index af0b1fb1dd4..c409b8a3410 100644 --- a/x-pack/filebeat/module/azure/auditlogs/config/auditlogs.yml +++ b/x-pack/filebeat/module/azure/auditlogs/config/auditlogs.yml @@ -2,7 +2,7 @@ type: kafka hosts: {{ .namespace }} topics: {{ .eventhub }} group_id: {{ .consumer_group }} -azure_logs: "AuditLogs" +log_types: "AuditLogs" username: "$ConnectionString" password: {{ .connection_string }} diff --git a/x-pack/filebeat/module/azure/signinlogs/config/signinlogs.yml b/x-pack/filebeat/module/azure/signinlogs/config/signinlogs.yml index 7a258346faf..4573beaf899 100644 --- a/x-pack/filebeat/module/azure/signinlogs/config/signinlogs.yml +++ b/x-pack/filebeat/module/azure/signinlogs/config/signinlogs.yml @@ -2,7 +2,7 @@ type: kafka hosts: {{ .namespace }} topics: {{ .eventhub }} group_id: {{ .consumer_group }} -azure_logs: "SigninLogs" +log_types: "SigninLogs" username: "$ConnectionString" password: {{ .connection_string }} From cb3ce2fc30e54287bb570f391fcd44cbbf0f56ac Mon Sep 17 00:00:00 2001 From: Mariana Date: Tue, 8 Oct 2019 14:08:44 +0200 Subject: [PATCH 13/21] Move azure log validation in pipelines --- filebeat/input/kafka/azure_log_patterns.go | 254 ------------------ filebeat/input/kafka/config.go | 41 ++- filebeat/input/kafka/input.go | 73 ++--- libbeat/common/kafka/logtype.go | 65 ----- .../activitylogs/config/activitylogs.yml | 4 +- .../azure/activitylogs/ingest/pipeline.json | 13 +- .../azure/auditlogs/config/auditlogs.yml | 4 +- .../azure/auditlogs/ingest/pipeline.json | 16 +- .../azure/signinlogs/config/signinlogs.yml | 4 +- .../azure/signinlogs/ingest/pipeline.json | 11 +- 10 files changed, 70 insertions(+), 415 deletions(-) delete mode 100644 filebeat/input/kafka/azure_log_patterns.go delete mode 100644 libbeat/common/kafka/logtype.go diff --git a/filebeat/input/kafka/azure_log_patterns.go b/filebeat/input/kafka/azure_log_patterns.go deleted file mode 100644 index 7998dd17f57..00000000000 --- a/filebeat/input/kafka/azure_log_patterns.go +++ /dev/null @@ -1,254 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package kafka - -import "time" - -// AzureActivityLogs structure matches the eventhub message carrying the azure activity logs -type AzureActivityLogs struct { - Records []ActivityLog `json:"records"` -} - -// AzureAuditLogs structure matches the eventhub message carrying the azure audit logs -type AzureAuditLogs struct { - Records []AuditLog `json:"records"` -} - -// AzureSigninLogs structure matches the eventhub message carrying the azure signin logs -type AzureSigninLogs struct { - Records []SigninLog `json:"records"` -} - -// ActivityLogs structure matches the azure activity log format -type ActivityLog struct { - Time time.Time `json:"time"` - ResourceID string `json:"resourceId"` - OperationName string `json:"operationName"` - Category string `json:"category"` - ResultType string `json:"resultType"` - ResultSignature string `json:"resultSignature"` - DurationMs int `json:"durationMs"` - CallerIPAddress string `json:"callerIpAddress"` - CorrelationID string `json:"correlationId"` - Identity struct { - Authorization struct { - Scope string `json:"scope"` - Action string `json:"action"` - Evidence struct { - Role string `json:"role"` - RoleAssignmentScope string `json:"roleAssignmentScope"` - RoleAssignmentID string `json:"roleAssignmentId"` - RoleDefinitionID string `json:"roleDefinitionId"` - PrincipalID string `json:"principalId"` - PrincipalType string `json:"principalType"` - } `json:"evidence"` - } `json:"authorization"` - Claims struct { - Aud string `json:"aud"` - Iss string `json:"iss"` - Iat string `json:"iat"` - Nbf string `json:"nbf"` - Exp string `json:"exp"` - Aio string `json:"aio"` - Appid string `json:"appid"` - Appidacr string `json:"appidacr"` - HTTPSchemasMicrosoftComIdentityClaimsIdentityprovider string `json:"http://schemas.microsoft.com/identity/claims/identityprovider"` - HTTPSchemasMicrosoftComIdentityClaimsObjectidentifier string `json:"http://schemas.microsoft.com/identity/claims/objectidentifier"` - HTTPSchemasXmlsoapOrgWs200505IdentityClaimsNameidentifier string `json:"http://schemas.xmlsoap.org/ws/2005/05/identity/claims/nameidentifier"` - HTTPSchemasMicrosoftComIdentityClaimsTenantid string `json:"http://schemas.microsoft.com/identity/claims/tenantid"` - Uti string `json:"uti"` - Ver string `json:"ver"` - } `json:"claims"` - } `json:"identity"` - Level string `json:"level"` - Location string `json:"location"` - Properties struct { - StatusCode string `json:"statusCode"` - ServiceRequestID interface{} `json:"serviceRequestId"` - StatusMessage string `json:"statusMessage"` - } `json:"properties,omitempty"` -} - -// AuditLog structure matches the azure audit log format -type AuditLog struct { - Category string `json:"category"` - CorrelationID string `json:"correlationId"` - DurationMs int `json:"durationMs"` - Level string `json:"level"` - OperationName string `json:"operationName"` - OperationVersion string `json:"operationVersion"` - Properties struct { - ActivityDateTime string `json:"activityDateTime"` - ActivityDisplayName string `json:"activityDisplayName"` - AdditionalDetails []interface{} `json:"additionalDetails"` - Category string `json:"category"` - CorrelationID string `json:"correlationId"` - ID string `json:"id"` - InitiatedBy struct { - App struct { - AppID interface{} `json:"appId"` - DisplayName string `json:"displayName"` - ServicePrincipalID string `json:"servicePrincipalId"` - ServicePrincipalName interface{} `json:"servicePrincipalName"` - } `json:"app"` - User struct { - DisplayName interface{} `json:"displayName"` - ID string `json:"id"` - IPAddress interface{} `json:"ipAddress"` - UserPrincipalName string `json:"userPrincipalName"` - } `json:"user"` - } `json:"initiatedBy"` - LoggedByService string `json:"loggedByService"` - OperationType string `json:"operationType"` - Result string `json:"result"` - ResultReason string `json:"resultReason"` - TargetResources []struct { - DisplayName interface{} `json:"displayName"` - ID string `json:"id"` - ModifiedProperties []struct { - DisplayName string `json:"displayName"` - NewValue string `json:"newValue"` - OldValue string `json:"oldValue"` - } `json:"modifiedProperties"` - Type string `json:"type"` - UserPrincipalName string `json:"userPrincipalName"` - } `json:"targetResources"` - } `json:"properties"` - ResourceID string `json:"resourceId"` - ResultSignature string `json:"resultSignature"` - TenantID string `json:"tenantId"` - Time string `json:"time"` - Identity string `json:"identity"` -} - -// SigninLog structure matches the azure audit log format -type SigninLog struct { - Level int `json:"Level"` - CallerIPAddress string `json:"callerIpAddress"` - Category string `json:"category"` - CorrelationID string `json:"correlationId"` - DurationMs int `json:"durationMs"` - Identity string `json:"identity"` - Location string `json:"location"` - OperationName string `json:"operationName"` - OperationVersion string `json:"operationVersion"` - Properties struct { - SignInBondData struct { - ConditionalAccessDetails interface{} `json:"ConditionalAccessDetails"` - DeviceDetails struct { - BrowserID interface{} `json:"BrowserId"` - BrowserType string `json:"BrowserType"` - DeviceDisplayName string `json:"DeviceDisplayName"` - DeviceID string `json:"DeviceId"` - DevicePlatform string `json:"DevicePlatform"` - DeviceTrustType int `json:"DeviceTrustType"` - IsCompliant interface{} `json:"IsCompliant"` - IsManaged interface{} `json:"IsManaged"` - UserAgent string `json:"UserAgent"` - } `json:"DeviceDetails"` - DisplayDetails struct { - ApplicationDisplayName string `json:"ApplicationDisplayName"` - AttemptedUsername interface{} `json:"AttemptedUsername"` - ProxyRestrictionTargetTenantName interface{} `json:"ProxyRestrictionTargetTenantName"` - ResourceDisplayName string `json:"ResourceDisplayName"` - UserName string `json:"UserName"` - } `json:"DisplayDetails"` - IssuerDetails interface{} `json:"IssuerDetails"` - LocationDetails struct { - IPChain interface{} `json:"IPChain"` - Latitude float64 `json:"Latitude"` - Longitude float64 `json:"Longitude"` - } `json:"LocationDetails"` - MfaDetails struct { - AuthMethod interface{} `json:"AuthMethod"` - MaskedDeviceID interface{} `json:"MaskedDeviceId"` - MfaStatus int `json:"MfaStatus"` - SasStatus interface{} `json:"SasStatus"` - } `json:"MfaDetails"` - PassThroughAuthenticationDetails interface{} `json:"PassThroughAuthenticationDetails"` - ProtocolDetails struct { - AuthenticationMethodsUsed interface{} `json:"AuthenticationMethodsUsed"` - DomainHintPresent interface{} `json:"DomainHintPresent"` - IsInteractive interface{} `json:"IsInteractive"` - LoginHintPresent interface{} `json:"LoginHintPresent"` - NetworkLocation interface{} `json:"NetworkLocation"` - Protocol interface{} `json:"Protocol"` - ResponseTime int `json:"ResponseTime"` - } `json:"ProtocolDetails"` - RAMDetails interface{} `json:"RamDetails"` - SourceAlpEvents interface{} `json:"SourceAlpEvents"` - } `json:"SignInBondData"` - AppDisplayName string `json:"appDisplayName"` - AppID string `json:"appId"` - AppliedConditionalAccessPolicies []interface{} `json:"appliedConditionalAccessPolicies"` - AuthenticationDetails []struct { - AuthenticationStepDateTime string `json:"authenticationStepDateTime"` - AuthenticationStepRequirement string `json:"authenticationStepRequirement"` - AuthenticationStepResultDetail string `json:"authenticationStepResultDetail"` - Succeeded bool `json:"succeeded"` - } `json:"authenticationDetails"` - AuthenticationProcessingDetails []interface{} `json:"authenticationProcessingDetails"` - AuthenticationRequirementPolicies []interface{} `json:"authenticationRequirementPolicies"` - ClientAppUsed string `json:"clientAppUsed"` - CorrelationID string `json:"correlationId"` - CreatedDateTime string `json:"createdDateTime"` - DeviceDetail struct { - DeviceID string `json:"deviceId"` - DisplayName string `json:"displayName"` - OperatingSystem string `json:"operatingSystem"` - TrustType string `json:"trustType"` - } `json:"deviceDetail"` - ID string `json:"id"` - IPAddress string `json:"ipAddress"` - IsInteractive bool `json:"isInteractive"` - Location struct { - City string `json:"city"` - CountryOrRegion string `json:"countryOrRegion"` - GeoCoordinates struct { - Latitude float64 `json:"latitude"` - Longitude float64 `json:"longitude"` - } `json:"geoCoordinates"` - State string `json:"state"` - } `json:"location"` - MfaDetail struct{} `json:"mfaDetail"` - NetworkLocationDetails []interface{} `json:"networkLocationDetails"` - OriginalRequestID string `json:"originalRequestId"` - ProcessingTimeInMilliseconds int `json:"processingTimeInMilliseconds"` - ResourceDisplayName string `json:"resourceDisplayName"` - ResourceID string `json:"resourceId"` - RiskDetail string `json:"riskDetail"` - RiskEventTypes []interface{} `json:"riskEventTypes"` - RiskLevelAggregated string `json:"riskLevelAggregated"` - RiskLevelDuringSignIn string `json:"riskLevelDuringSignIn"` - RiskState string `json:"riskState"` - Status struct { - AdditionalDetails string `json:"additionalDetails"` - ErrorCode int `json:"errorCode"` - } `json:"status"` - TokenIssuerName string `json:"tokenIssuerName"` - TokenIssuerType string `json:"tokenIssuerType"` - UserDisplayName string `json:"userDisplayName"` - UserID string `json:"userId"` - UserPrincipalName string `json:"userPrincipalName"` - } `json:"properties"` - ResourceID string `json:"resourceId"` - ResultSignature string `json:"resultSignature"` - ResultType string `json:"resultType"` - TenantID string `json:"tenantId"` - Time string `json:"time"` -} diff --git a/filebeat/input/kafka/config.go b/filebeat/input/kafka/config.go index 039d6c43a8c..e563248bddf 100644 --- a/filebeat/input/kafka/config.go +++ b/filebeat/input/kafka/config.go @@ -34,23 +34,23 @@ import ( type kafkaInputConfig struct { // Kafka hosts with port, e.g. "localhost:9092" - Hosts []string `config:"hosts" validate:"required"` - Topics []string `config:"topics" validate:"required"` - GroupID string `config:"group_id" validate:"required"` - ClientID string `config:"client_id"` - Version kafka.Version `config:"version"` - InitialOffset initialOffset `config:"initial_offset"` - ConnectBackoff time.Duration `config:"connect_backoff" validate:"min=0"` - ConsumeBackoff time.Duration `config:"consume_backoff" validate:"min=0"` - WaitClose time.Duration `config:"wait_close" validate:"min=0"` - MaxWaitTime time.Duration `config:"max_wait_time"` - IsolationLevel isolationLevel `config:"isolation_level"` - Fetch kafkaFetch `config:"fetch"` - Rebalance kafkaRebalance `config:"rebalance"` - TLS *tlscommon.Config `config:"ssl"` - Username string `config:"username"` - Password string `config:"password"` - LogType kafka.LogType `config:"log_types"` + Hosts []string `config:"hosts" validate:"required"` + Topics []string `config:"topics" validate:"required"` + GroupID string `config:"group_id" validate:"required"` + ClientID string `config:"client_id"` + Version kafka.Version `config:"version"` + InitialOffset initialOffset `config:"initial_offset"` + ConnectBackoff time.Duration `config:"connect_backoff" validate:"min=0"` + ConsumeBackoff time.Duration `config:"consume_backoff" validate:"min=0"` + WaitClose time.Duration `config:"wait_close" validate:"min=0"` + MaxWaitTime time.Duration `config:"max_wait_time"` + IsolationLevel isolationLevel `config:"isolation_level"` + Fetch kafkaFetch `config:"fetch"` + Rebalance kafkaRebalance `config:"rebalance"` + TLS *tlscommon.Config `config:"ssl"` + Username string `config:"username"` + Password string `config:"password"` + YieldEventsFromField string `config:"yield_events_from_field"` } type kafkaFetch struct { @@ -138,13 +138,6 @@ func (c *kafkaInputConfig) Validate() error { return err } - // if any log types are specified by the metricset config then a validation of the type will be done, else, no extra processing will happen to the messages - if c.LogType != "" { - if err := c.LogType.Validate(); err != nil { - return err - } - } - if c.Username != "" && c.Password == "" { return fmt.Errorf("password must be set when username is configured") } diff --git a/filebeat/input/kafka/input.go b/filebeat/input/kafka/input.go index 23c12a74b3d..75a8a1ee7a4 100644 --- a/filebeat/input/kafka/input.go +++ b/filebeat/input/kafka/input.go @@ -109,8 +109,10 @@ func (input *kafkaInput) runConsumerGroup( handler := &groupHandler{ version: input.config.Version, outlet: input.outlet, - // logType will be assigned the configuration option AzureLogs, if the metricset using this input is an azure metricset then we can filter for the azure log types dedicated to the metricset (audit, activity, signin) - logType: input.config.LogType, + // yieldEventsFromField will be assigned the configuration option yield_events_from_field, + // if the fileset using this input expects to receive multiple messages bundled under a specific field then this value is assigned + // ex. in this case are the azure fielsets where the events are found under the json object "records" + yieldEventsFromField: input.config.YieldEventsFromField, } input.saramaWaitGroup.Add(1) @@ -235,10 +237,10 @@ func (c channelCtx) Value(key interface{}) interface{} { return nil } // and passing ACKs from the output channel back to the kafka cluster. type groupHandler struct { sync.Mutex - version kafka.Version - session sarama.ConsumerGroupSession - outlet channel.Outleter - logType kafka.LogType + version kafka.Version + session sarama.ConsumerGroupSession + outlet channel.Outleter + yieldEventsFromField string } // The metadata attached to incoming events so they can be ACKed once they've @@ -275,7 +277,13 @@ func (h *groupHandler) createEvent( // if azure input, then a check for the message is done regarding the list of events var events []beat.Event - messages := h.parseMultipleMessages(message.Value) + var messages []string + // if no values are ser + if h.yieldEventsFromField == "" { + messages = []string{string(message.Value)} + } else { + messages = h.parseMultipleMessages(message.Value) + } for _, msg := range messages { event := beat.Event{ Timestamp: timestamp, @@ -328,56 +336,21 @@ func (h *groupHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sara return nil } +// parseMultipleMessages will try to split the message into multiple ones based on the group field provided by the configuration func (h *groupHandler) parseMultipleMessages(bMessage []byte) []string { var messages []string - // check if logType has been set - switch h.logType { - // if no log type has been set then the original message should be returned - case "": - default: - return []string{string(bMessage)} - case kafka.AuditLogs: - // if the fileset is audit logs a filtering of the messages should be done as the eventhub can return different types of messages - var obj AzureAuditLogs - err := json.Unmarshal(bMessage, &obj) - if err != nil { - return nil - } - for _, ms := range obj.Records { - js, err := json.Marshal(ms) - if err == nil { - messages = append(messages, string(js)) - } - } - return messages - case kafka.ActivityLogs: - // if the fileset is activity logs a filtering of the messages should be done as the eventhub can return different types of messages - var obj AzureActivityLogs - err := json.Unmarshal(bMessage, &obj) - if err != nil { - return nil - } - for _, ms := range obj.Records { - js, err := json.Marshal(ms) - if err == nil { - messages = append(messages, string(js)) - } - } + var obj map[string][]interface{} + err := json.Unmarshal(bMessage, &obj) + if err != nil { return messages - case kafka.SigninLogs: - // if the fileset is signin logs a filtering of the messages should be done as the eventhub can return different types of messages - var obj AzureSigninLogs - err := json.Unmarshal(bMessage, &obj) - if err != nil { - return nil - } - for _, ms := range obj.Records { + } + if len(obj[h.yieldEventsFromField]) > 0 { + for _, ms := range obj[h.yieldEventsFromField] { js, err := json.Marshal(ms) if err == nil { messages = append(messages, string(js)) } } - return messages } - return nil + return messages } diff --git a/libbeat/common/kafka/logtype.go b/libbeat/common/kafka/logtype.go deleted file mode 100644 index b45cf0fa95f..00000000000 --- a/libbeat/common/kafka/logtype.go +++ /dev/null @@ -1,65 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package kafka - -import ( - "fmt" -) - -// LogType is the type of the messages coming in -type LogType string - -var ( - // ActivityLogs variable used to identify the activitylogs azure metricset - ActivityLogs LogType = "ActivityLogs" - // AuditLogs variable used to identify the auditlogs azure metricset - AuditLogs LogType = "AuditLogs" - // SigninLogs variable used to identify the signinlogs azure metricset - SigninLogs LogType = "SigninLogs" - - LogTypes = map[string]LogType{ - "AuditLogs": AuditLogs, - "ActivityLogs": ActivityLogs, - "SigninLogs": SigninLogs, - } -) - -// Validate that a log type is among the possible options -func (lt *LogType) Validate() error { - if _, ok := LogTypes[string(*lt)]; !ok { - return fmt.Errorf("unknown/unsupported kafka vesion '%v'", *lt) - } - - return nil -} - -// Unpack the log type version -func (lt *LogType) Unpack(s string) error { - tmp := LogType(s) - if err := tmp.Validate(); err != nil { - return err - } - *lt = tmp - return nil -} - -// Get the log type -func (lt LogType) Get() (LogType, bool) { - kv, ok := LogTypes[string(lt)] - return kv, ok -} diff --git a/x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml b/x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml index 0018b039f62..edcdfb94723 100644 --- a/x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml +++ b/x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml @@ -2,7 +2,7 @@ type: kafka hosts: {{ .namespace }} topics: {{ .eventhub }} group_id: {{ .consumer_group }} -log_types: "ActivityLogs" +yield_events_from_field: "records" username: "$ConnectionString" password: {{ .connection_string }} @@ -11,4 +11,4 @@ ssl.enabled: true processors: - decode_json_fields: fields: ["message"] - target: "" + target: "azure.activitylogs" diff --git a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json index b5c2241a5e4..0b3bbc9596e 100644 --- a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json +++ b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json @@ -3,7 +3,7 @@ "processors": [ { "date": { - "field": "time", + "field": "azure.activitylogs.time", "target_field": "@timestamp", "ignore_failure": false, "formats": [ @@ -13,16 +13,21 @@ }, { "rename": { - "field": "resourceId", - "target_field": "resourceID", + "field": "azure.activitylogs.resourceId", + "target_field": "azure.activitylogs.resourceID", "ignore_missing": true } }, { "remove": { - "field": ["message", "time"], + "field": ["message", "azure.activitylogs.time"], "ignore_missing": true } + }, + { + "drop": { + "if" : "ctx.azure.activitylogs.identity == null || ctx.azure.activitylogs.identity.claims == null" + } } ], "on_failure": [ diff --git a/x-pack/filebeat/module/azure/auditlogs/config/auditlogs.yml b/x-pack/filebeat/module/azure/auditlogs/config/auditlogs.yml index c409b8a3410..4221a297987 100644 --- a/x-pack/filebeat/module/azure/auditlogs/config/auditlogs.yml +++ b/x-pack/filebeat/module/azure/auditlogs/config/auditlogs.yml @@ -2,7 +2,7 @@ type: kafka hosts: {{ .namespace }} topics: {{ .eventhub }} group_id: {{ .consumer_group }} -log_types: "AuditLogs" +yield_events_from_field: "records" username: "$ConnectionString" password: {{ .connection_string }} @@ -11,4 +11,4 @@ ssl.enabled: true processors: - decode_json_fields: fields: ["message"] - target: "" + target: "azure.auditlogs" diff --git a/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.json b/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.json index f394b9cece5..5cd94946113 100644 --- a/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.json +++ b/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.json @@ -3,7 +3,7 @@ "processors": [ { "date": { - "field": "time", + "field": "azure.auditlogs.time", "target_field": "@timestamp", "ignore_failure": false, "formats": [ @@ -13,22 +13,20 @@ }, { "rename": { - "field": "resourceId", - "target_field": "resourceID", + "field": "azure.auditlogs.resourceId", + "target_field": "azure.auditlogs.resourceID", "ignore_missing": true } }, { - "rename": { - "field": "identity", - "target_field": "specification", + "remove": { + "field": ["message", "azure.auditlogs.time"], "ignore_missing": true } }, { - "remove": { - "field": ["message", "time"], - "ignore_missing": true + "drop": { + "if" : "ctx.azure.auditlogs.category != 'AuditLogs'" } } ], diff --git a/x-pack/filebeat/module/azure/signinlogs/config/signinlogs.yml b/x-pack/filebeat/module/azure/signinlogs/config/signinlogs.yml index 4573beaf899..a87b87bf7bb 100644 --- a/x-pack/filebeat/module/azure/signinlogs/config/signinlogs.yml +++ b/x-pack/filebeat/module/azure/signinlogs/config/signinlogs.yml @@ -2,7 +2,7 @@ type: kafka hosts: {{ .namespace }} topics: {{ .eventhub }} group_id: {{ .consumer_group }} -log_types: "SigninLogs" +yield_events_from_field: "records" username: "$ConnectionString" password: {{ .connection_string }} @@ -11,4 +11,4 @@ ssl.enabled: true processors: - decode_json_fields: fields: ["message"] - target: "" + target: "azure.signinlogs" diff --git a/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.json b/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.json index 08a42caa448..c0e82be7c28 100644 --- a/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.json +++ b/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.json @@ -3,7 +3,7 @@ "processors": [ { "date": { - "field": "time", + "field": "azure.signinlogs.time", "target_field": "@timestamp", "ignore_failure": false, "formats": [ @@ -13,8 +13,8 @@ }, { "rename": { - "field": "resourceId", - "target_field": "resourceID", + "field": "azure.signinlogs.resourceId", + "target_field": "azure.signinlogs.resourceID", "ignore_missing": true } }, @@ -30,6 +30,11 @@ "field": ["message", "time"], "ignore_missing": true } + }, + { + "drop": { + "if" : "ctx.azure.signinlogs.category != 'SignInLogs'" + } } ], "on_failure": [ From f2a55184596368f06b23844f2d49b74551688804 Mon Sep 17 00:00:00 2001 From: Mariana Date: Tue, 8 Oct 2019 15:57:24 +0200 Subject: [PATCH 14/21] Work on event format --- filebeat/docs/fields.asciidoc | 100 ------------------ .../azure/activitylogs/_meta/fields.yml | 16 --- .../azure/activitylogs/ingest/pipeline.json | 38 ++++++- .../module/azure/auditlogs/_meta/fields.yml | 8 -- .../azure/auditlogs/ingest/pipeline.json | 16 ++- x-pack/filebeat/module/azure/fields.go | 2 +- .../module/azure/signinlogs/_meta/fields.yml | 17 --- .../azure/signinlogs/ingest/pipeline.json | 27 ++++- 8 files changed, 77 insertions(+), 147 deletions(-) diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 9d1023ab914..76364b1009e 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -1389,16 +1389,6 @@ type: keyword Result Type -type: keyword - --- - -*`azure.activitylogs.callerIpAddress`*:: -+ --- -Caller Ip address - - type: keyword -- @@ -1409,16 +1399,6 @@ type: keyword Resource ID -type: keyword - --- - -*`azure.activitylogs.level`*:: -+ --- -Level - - type: keyword -- @@ -1453,26 +1433,6 @@ type: object -- -*`azure.activitylogs.location`*:: -+ --- -Location - - -type: keyword - --- - -*`azure.activitylogs.category`*:: -+ --- -Category - - -type: keyword - --- - [float] === auditlogs @@ -1536,16 +1496,6 @@ type: keyword Result signature -type: keyword - --- - -*`azure.auditlogs.durationMs`*:: -+ --- -Activity duration - - type: keyword -- @@ -1556,16 +1506,6 @@ type: keyword Correlation ID -type: keyword - --- - -*`azure.auditlogs.level`*:: -+ --- -Log level - - type: keyword -- @@ -1925,16 +1865,6 @@ type: keyword Result signature -type: keyword - --- - -*`azure.signinlogs.durationMs`*:: -+ --- -Activity duration - - type: keyword -- @@ -1945,16 +1875,6 @@ type: keyword Correlation ID -type: keyword - --- - -*`azure.signinlogs.level`*:: -+ --- -Log level - - type: keyword -- @@ -1965,16 +1885,6 @@ type: keyword Result type -type: keyword - --- - -*`azure.signinlogs.callerIpAddress`*:: -+ --- -Caller Ip address - - type: keyword -- @@ -1985,16 +1895,6 @@ type: keyword Identity -type: keyword - --- - -*`azure.signinlogs.location`*:: -+ --- -Location - - type: keyword -- diff --git a/x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml b/x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml index d4c120cdb98..cc3f3036649 100644 --- a/x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml +++ b/x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml @@ -64,18 +64,10 @@ type: keyword description: > Result Type - - name: callerIpAddress - type: keyword - description: > - Caller Ip address - name: resourceID type: keyword description: > Resource ID - - name: level - type: keyword - description: > - Level - name: operationName type: keyword description: > @@ -90,12 +82,4 @@ object_type_mapping_type: "*" description: > Properties - - name: location - type: keyword - description: > - Location - - name: category - type: keyword - description: > - Category diff --git a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json index 0b3bbc9596e..2e841e24688 100644 --- a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json +++ b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json @@ -18,9 +18,45 @@ "ignore_missing": true } }, + { + "rename": { + "field": "azure.activitylogs.callerIpAddress", + "target_field": "source.ip", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.activitylogs.level", + "target_field": "log.level", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.activitylogs.durationMs", + "target_field": "event.duration", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.activitylogs.location", + "target_field": "geo.name", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.activitylogs.category", + "target_field": "event.category", + "ignore_missing": true + } + }, { "remove": { - "field": ["message", "azure.activitylogs.time"], + "field": ["message", "azure.activitylogs.time", "azure.activitylogs.callerIpAddress", "azure.activitylogs.category", + "azure.activitylogs.level", "azure.activitylogs.durationMs", "azure.activitylogs.location"], "ignore_missing": true } }, diff --git a/x-pack/filebeat/module/azure/auditlogs/_meta/fields.yml b/x-pack/filebeat/module/azure/auditlogs/_meta/fields.yml index 4f0fa2df629..c76e45b11a6 100644 --- a/x-pack/filebeat/module/azure/auditlogs/_meta/fields.yml +++ b/x-pack/filebeat/module/azure/auditlogs/_meta/fields.yml @@ -27,18 +27,10 @@ type: keyword description: > Result signature - - name: durationMs - type: keyword - description: > - Activity duration - name: correlationId type: keyword description: > Correlation ID - - name: level - type: keyword - description: > - Log level - name: properties type: group description: > diff --git a/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.json b/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.json index 5cd94946113..33badd8ee2d 100644 --- a/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.json +++ b/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.json @@ -18,9 +18,23 @@ "ignore_missing": true } }, + { + "rename": { + "field": "azure.auditlogs.durationMs", + "target_field": "event.duration", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.auditlogs.level", + "target_field": "log.level", + "ignore_missing": true + } + }, { "remove": { - "field": ["message", "azure.auditlogs.time"], + "field": ["message", "azure.auditlogs.time", "azure.auditlogs.durationMs", "azure.auditlogs.level"], "ignore_missing": true } }, diff --git a/x-pack/filebeat/module/azure/fields.go b/x-pack/filebeat/module/azure/fields.go index 153c412a23e..2a2bdeb5ace 100644 --- a/x-pack/filebeat/module/azure/fields.go +++ b/x-pack/filebeat/module/azure/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAzure returns asset data. // This is the base64 encoded gzipped contents of module/azure. func AssetAzure() string { - return "eJzsW81u2zgQvucpBjkWaB8ghwVce3dhoGmK/PRaMNJE4YYmtSTlwH36BSnrxxYp0TFpZ4Ho1FTKfB/JGc7HGeYzvODmCsjvSuIFgKaa4RVc2p8vLwByVJmkpaaCX8EfFwBQfwvXIq+Y+ZUniixXV/bVZ+BkhZ058+hNiVdQSFGV2/9x2Nw1s2Mq03RN9YaJQrUvXVa9luvnL2sfnoTcDmC2NQzG8pfet/tM+mxojlxTvdl56WMzwcg8988IGeGC04wwqBRKWC5APIF+RhCvHGXzgxKVzBAeq+wF9Zc9Qy7KfdoZI3SlvnwafNBQF4//YKYdr+sXv+qvXnDzKmQ+/tmvFSlLyovt71x+unR8PzEt5plbzt4hkUo/C0l/E2PDOy7XkgTifxc5AmFMZESTR4ZQinxIxz/3fbK4No6TofOjKa6BfA/hPMW7z10KhjOlaMFXyPVdJkrfMLqh+B3lwAGZ51YwBNIyADVKoU97gU+UU4OxHOOSinPewsNyEUT49CQPXP6zzGNv7QPmsZSUZ7Qk7NRcfzTAB7G835w6njqe2oXdMPSF2RSjACbuTaSf8Ue29KOQZ0PbbY4UUiIjzt1iDHkCdd6Z3feLNsRQVUw7HOEI2FtrEwZG29ESxlAuy1meS1T7OeKY8VrDsCyBOEz3hmz1zCBSjhtyrZE808xwjSwe3LeBuQZIlCjtgn8nq4hLetOYtTAjnnRHC050p8AjYG/dSTktd3uaGbmmOFSaIyozQGEeoi4nBvKj5ej2EaOchrvPMW7isthFocZCSPdx4o3ht2dxeKCqcqqjn6as1YOOUufYAhJFpjnFienobD/5iVJFdbJdAmuH+XQe900UbqsNpEZOHMrxmOFai+Pp9FybYF7Vq3AdMae2pYrG+DsTMLEzqygcJodZxon4xiJMu3/57U8VWWq/S6NbzZR47O8XyhZUlYxsHDtcJC6dN9ZIww1vOCu3SFQqTV/btrlIP6Ndw4mpGouUSKRGogZ29EZRYP51c4dyTZ31oTjOU2AOjxtQHphBhvKcSiOQ6VTs6OmTJlqXkbVoA4hovKfJo0fTFSpNVsOS30SujkTEa73N20QWqBtJ5SojHl1jvbcQrRR8a3k1H93yILCMElhCWYztejDlwpHJeCpN7RL6i0sRSXhLWO1MeMoMCbgALWEMrKFUKZRtIew0jvOgUHZ1v2n/WYmcPlHMPUfWfY4R2gfXW0S/Bmqe0OYBx9efhFUnrnB+x1dYj8KG7xyJKDa7SBhNwfIzTOMNyz382sDmVFOijYJJkR6W/EnIVa0WJBZE5pQXVuRtgYV8Y8YgpS9UogXTzAsRGjxbtRayTUEaD9jK0pBdC95TVAWxJGV56hbRrCxDmkP7C39qmsNln9AZJpumjqcHP0ZoQIUmfUgzraECAP53oeSVuonIBcTQlOZMxMwhP0P+0UY+LTjlH9X5j+p8JOyP6vwO+kd1/mjE8ep8wrsMg2LGO7nLMHon9E24S5fFM7Wo07Rb6jx3XL/lDCXiTKI57aatEM9rEMiJRlsl9tIxejZ5u8eq1qBWz/tpq4QI/VhTMyHo+5RSzcpDfW/ci+076cZoZ7hPtD3k9P3Isgzzz7EjQYx9w50mYOcePkWuZ2X5oDBVhFgIM/HG5YZGumDlub2bTNgsy1CpO010lWhq5h0YEIsGyg3XqnFJC8oJu8V/K1Tue8cxupFbGJA1zpgvU7XkGqVtEyZy5aUCOoLRanbxgnypVIUyXVjdGxCgFmU8rnp00vWNd+iMto5LKYyHUV6YHL3k15QxqtA4fCL3/tEi2nwNlMNqDLQVy1S9LFATui/PI/G6peoFcjdAn4O9yjorComFUR4JydiDAxA/1IDWopKUF+a8uEx1f6Qjllswq04daH1uZrdM5OmWj3La36/OJM+tbckmKMF69vQInSBvcgr66ysphZyL/AQ93j8NFGQurLYEgWua4UTYn2uqanLefkPMaxQWaaqxsC2b8eJuozSu0tO6aQBB+REbdo9SvE51PqKw+uoBmqgJwNn9KRvWRXaJRZmeuQ+lvzGdIPp9SaGvuiuu5eZG3mLhXq/IlOY1IAgJ0g/Z0CtQzIWQOeVEp79u8ncIWmhvjxFNdeXd4yFNu+fbFGoXobw4B78B7MV/AQAA//+Kx76S" + return "eJzsW81u2zgQvvcpBjkWaB8ghwVce3dhoG2K/PRaMOJE4YYmtSTlwH36BSmLUiJSYmLR7hbRqamU+T4OZ4Yfh8wHeMDdOZCftcJ3AIYZjudw5n4+ewdAUReKVYZJcQ5/vAOA5lv4ImnN7a/cMeRUn7tXH0CQDXbm7GN2FZ5DqWRd7f8nYPOpmSemCsO2zOy4LLV/GbIatdw8fzn7cCfVfgCLvWGwlj/2vn3OpM+GURSGmd2TlzE2E4zsc32PUBAhBSsIh1qjgvUK5B2YewT5KFC1P2hZqwLhti4e0Hx8ZihEuU+74IRt9Mf3gw9a6vL2HyxM4HXz4kfz1QPuHqWi45/92JCqYqLc/87Z+7PA9xNusc/ScY4OidTmXir2k1gb0XGFpiQR/6ukCIRzWRBDbjlCJemQTtz3fbK4tYFTYPCjKa6JfF/CeYp3n7uSHBdas1JsUJirQlaxYXRDiQfKCwdkn0vJEYhnAHqUQp/2Cu+YYBZjPcYlF2fq4WG9SiJ8fJIvnP6T+LE39wl+rBQTBasIPzbXby3wi1he746dTx1PE8JuGcbSbIpRApNwEemv+CMl/SDkxdC2XyOlUshJsFqMIU+gLjuzz+PCpxjqmptAIBwAe+lswsBoD9MJikGoHobZiJTIOGWFyjniK9nMONSL1qyDGfHwFSsFMZ0ynWfI1s06aLnLdTtyw3CowEbUV4LyeonqmhjIN8/RvxpK8ZoyM7sOd1ZfJMJ/o9i1+l9Ox6//5DsqPSyOsxHYBsz7+kgMllKFtz6vwv4sy7DVFtKgIAHNcchwncXxQnyqMnG6BagaJn8f9pX7Wp/YcftT+9ZmQvJIARt9EfvPew8rpitOdoHUn4mL70XQBmlYCYZeuUSic8mkxrYr0uYe3RxOuGoseGciNRLIfSZcliXST7srVFsW3HLPEzwlUrjdgY7ADEp3ROjPQKYTQKOCnmWal5G58AlEDF6z7Nlj2Aa1IZthF2ViEZuJSNS6X9CIKtG0WiPUmTm4bXXtILxGem3Hio6WPEjcmSbuSldjVQ+mQnhmMpHNu5/C+H59RhLRroD3RLWgVKGOdfdm5AKsgjGwllKtUfnewnEC50aj6lop0/GzkZTdMaSB3U6I4wwd2S97xLgGap/UfqzAx++E10duGn3FR9iOwqZXjkwU2yqSRlNyegI3XnAa4ecTWzDDiLEKJsfysBZ3Um0ataCwJIoyUTqRtweW6pUrBqliqTJbMi2iEKnJs1drKWUK8kTAXpamVC34lbIqiSWpqmN33RdVldJvfz7xx6Y5nPYJnWFX09z5dBPHSE2o1EUf8rg1VQDA/y6VolI3E7mEHJrSnJmYBeRnyj985rNSMPHWtn5rW8+E/da2foL+W56bDnb5SRfAXoW4DlnM25lvSuJhrfkTdBMLhXZjlLeZuGxAgBKDrqEYpWOlT/aTASdwkk4Ffp0OfIomnMs1E9qvTymXV26aW5tR7NimaI7Od3jz00POf3RVVWnxOaYe56gbFZCI/e4WLENhFlV1ozFXhjgI63gbckMjXbIK6m4GEr4oCtT6yhBTZ3LNsgMD4tBAh+G8cFOsZILwS/y3Rh2+9TfHwdUeBlSDMxbLTK+FQeVOlDKF8loDG8Hw8k4+oFhrXaPKl1bXFgSYQxnPqx6dfEeMT+iMnjJWStoIY6K0a/RafGGcM4024DOF9zeP6NZrYAI2Y6BePjL9sEJDGM/D65LpB6BhgD6Hz7hFvihLhaVVHhnJcIsEJA41oLWqFROl3Vqsc1016IhRB+bUaQCtz81Wy0yR7vjooP3nG/nsa6vf3SctsJGaPsOhQXRxSvrbB6WkWkp6hOPAPy0UFCEs3/zDLStwIu1P5aqGXLQ1PeeJu0Oa6kHvOyyivNppg5v8tC5aQNBxxJbdrZKPU03yWVh9igB1N46KbH+FdFA8FcNGxVNis7hnGUPpF6YjZH9sUeir7loYtbtQl1iG52tmSssGEKQCFYds6ZUol1IqygQx+W8m/J2ClnoMxIlhpo7WeMhzMvB5CrXLUFGegp+H/S8AAP//oxWmiQ==" } diff --git a/x-pack/filebeat/module/azure/signinlogs/_meta/fields.yml b/x-pack/filebeat/module/azure/signinlogs/_meta/fields.yml index 22f8f48040a..dfabc236891 100644 --- a/x-pack/filebeat/module/azure/signinlogs/_meta/fields.yml +++ b/x-pack/filebeat/module/azure/signinlogs/_meta/fields.yml @@ -27,34 +27,18 @@ type: keyword description: > Result signature - - name: durationMs - type: keyword - description: > - Activity duration - name: correlationId type: keyword description: > Correlation ID - - name: level - type: keyword - description: > - Log level - name: resultType type: keyword description: > Result type - - name: callerIpAddress - type: keyword - description: > - Caller Ip address - name: identity type: keyword description: > Identity - - name: location - type: keyword - description: > - Location - name: properties type: group description: > @@ -200,4 +184,3 @@ type: keyword description: > Longitude - diff --git a/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.json b/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.json index c0e82be7c28..72c6123c81b 100644 --- a/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.json +++ b/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.json @@ -20,14 +20,35 @@ }, { "rename": { - "field": "identity", - "target_field": "specification", + "field": "azure.signinlogs.callerIpAddress", + "target_field": "source.ip", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.Level", + "target_field": "log.level", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.durationMs", + "target_field": "event.duration", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.location", + "target_field": "geo.country_iso_code", "ignore_missing": true } }, { "remove": { - "field": ["message", "time"], + "field": ["message", "azure.signinlogs.time","azure.signinlogs.Level", "azure.signinlogs.callerIpAddress","azure.signinlogs.durationMs"], "ignore_missing": true } }, From ff26f0456f59f8d4b42eeea1ea03e8353d1e3e51 Mon Sep 17 00:00:00 2001 From: Mariana Date: Tue, 8 Oct 2019 18:33:22 +0200 Subject: [PATCH 15/21] Work on activitylogs format --- filebeat/docs/fields.asciidoc | 10 -- filebeat/docs/modules/azure.asciidoc | 11 +- .../filebeat/module/azure/_meta/docs.asciidoc | 11 +- .../azure/activitylogs/_meta/fields.yml | 4 - .../azure/activitylogs/ingest/pipeline.json | 9 +- .../test/activity_log_expected.json | 146 ++++++++++-------- x-pack/filebeat/module/azure/fields.go | 2 +- .../azure/signinlogs/ingest/pipeline.json | 17 +- 8 files changed, 107 insertions(+), 103 deletions(-) diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 76364b1009e..46a146996c0 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -1379,16 +1379,6 @@ type: keyword Correlation ID -type: keyword - --- - -*`azure.activitylogs.resultType`*:: -+ --- -Result Type - - type: keyword -- diff --git a/filebeat/docs/modules/azure.asciidoc b/filebeat/docs/modules/azure.asciidoc index 9b51551e2e0..d428fb6f382 100644 --- a/filebeat/docs/modules/azure.asciidoc +++ b/filebeat/docs/modules/azure.asciidoc @@ -3,6 +3,8 @@ This file is generated! See scripts/docs_collector.py //// [[filebeat-module-azure]] +[role="xpack"] + :modulename: azure :has-dashboards: false @@ -87,15 +89,6 @@ include::../include/what-happens.asciidoc[] TODO: document with what versions of the software is this tested -include::../include/running-modules.asciidoc[] - -[float] -=== Example dashboard - -This module comes with a sample dashboard. For example: - -TODO: include an image of a sample dashboard. If you do not include a dashboard, -remove this section and set `:has-dashboards: false` at the top of this file. diff --git a/x-pack/filebeat/module/azure/_meta/docs.asciidoc b/x-pack/filebeat/module/azure/_meta/docs.asciidoc index d352cdfb4c0..33d84149d08 100644 --- a/x-pack/filebeat/module/azure/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/azure/_meta/docs.asciidoc @@ -1,3 +1,5 @@ +[role="xpack"] + :modulename: azure :has-dashboards: false @@ -82,15 +84,6 @@ include::../include/what-happens.asciidoc[] TODO: document with what versions of the software is this tested -include::../include/running-modules.asciidoc[] - -[float] -=== Example dashboard - -This module comes with a sample dashboard. For example: - -TODO: include an image of a sample dashboard. If you do not include a dashboard, -remove this section and set `:has-dashboards: false` at the top of this file. diff --git a/x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml b/x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml index cc3f3036649..ed5cd065446 100644 --- a/x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml +++ b/x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml @@ -60,10 +60,6 @@ type: keyword description: > Correlation ID - - name: resultType - type: keyword - description: > - Result Type - name: resourceID type: keyword description: > diff --git a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json index 2e841e24688..9ac260d8cb3 100644 --- a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json +++ b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json @@ -53,10 +53,17 @@ "ignore_missing": true } }, + { + "rename": { + "field": "azure.activitylogs.resultType", + "target_field": "event.outcome", + "ignore_missing": true + } + }, { "remove": { "field": ["message", "azure.activitylogs.time", "azure.activitylogs.callerIpAddress", "azure.activitylogs.category", - "azure.activitylogs.level", "azure.activitylogs.durationMs", "azure.activitylogs.location"], + "azure.activitylogs.level", "azure.activitylogs.durationMs", "azure.activitylogs.location", "azure.activitylogs.resultType"], "ignore_missing": true } }, diff --git a/x-pack/filebeat/module/azure/activitylogs/test/activity_log_expected.json b/x-pack/filebeat/module/azure/activitylogs/test/activity_log_expected.json index 766b5dda06a..c32ebc37934 100644 --- a/x-pack/filebeat/module/azure/activitylogs/test/activity_log_expected.json +++ b/x-pack/filebeat/module/azure/activitylogs/test/activity_log_expected.json @@ -1,5 +1,5 @@ { - "_index" : "filebeat-8.0.0-2019.09.26", + "_index" : "filebeat-8.0.0-2019.10.08", "_type" : "_doc", "_id" : "2a7e2503-d7e2", "_score" : null, @@ -11,44 +11,42 @@ "ephemeral_id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", "version" : "8.0.0" }, + "log" : { + "level" : "Error" + }, + "source" : { + "ip" : "51.161.101.191" + }, + "fileset" : { + "name" : "activitylogs" + }, + "geo" : { + "name" : "global" + }, + "input" : { + "type" : "kafka" + }, + "@timestamp" : "2019-10-08T15:24:49.119Z", "ecs" : { "version" : "1.1.0" }, - "identity" : { - "authorization" : { - "evidence" : { - "roleAssignmentScope" : "/subscriptions/2a7e2503-d7e2-405a-a84c-c333b9f7cb73", - "roleDefinitionId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", - "role" : "Azure EventGrid Service BuiltIn Role", - "roleAssignmentId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", - "principalId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", - "principalType" : "ServicePrincipal" - }, - "scope" : "/subscriptions/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/resourceGroups/sa-hemant/providers/Microsoft.EventHub/namespaces/azurelsevents/authorizationRules/RootManageSharedAccessKey", - "action" : "Microsoft.EventHub/namespaces/authorizationRules/listKeys/action" - }, - "claims" : { - "http://schemas.xmlsoap.org/ws/2005/05/identity/claims/nameidentifier" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", - "ver" : "1.0", - "http://schemas.microsoft.com/identity/claims/identityprovider" : "https://sts.windows.net/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/", - "aio" : "2a7e2503-d7e2-40", - "iss" : "https://sts.windows.net/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/", - "uti" : "2a7e2503-d7e2", - "aud" : "https://management.core.windows.net/", - "nbf" : "1569500393", - "appidacr" : "2", - "http://schemas.microsoft.com/identity/claims/tenantid" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", - "appid" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", - "http://schemas.microsoft.com/identity/claims/objectidentifier" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", - "exp" : "1569529493", - "iat" : "1569500393" - } + "service" : { + "type" : "azure" + }, + "kafka" : { + "headers" : [ + "ProfileName: �\fname=default" + ], + "partition" : 3, + "offset" : 44768, + "topic" : "insights-operational-logs", + "key" : "" }, "host" : { "hostname" : "DESKTOP-RFOOE09", "os" : { - "build" : "17134.1006", - "kernel" : "10.0.17134.1006 (WinBuild.160101.0800)", + "build" : "18362.388", + "kernel" : "10.0.18362.388 (WinBuild.160101.0800)", "name" : "Windows 10 Pro", "family" : "windows", "version" : "10.0", @@ -58,46 +56,58 @@ "id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", "architecture" : "x86_64" }, - "correlationId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", "event" : { + "duration" : 11, "module" : "azure", - "dataset" : "azure.activitylogs" - }, - "resultType" : "Failure", - "callerIpAddress" : "11.71.131.141", - "resourceID" : "/SUBSCRIPTIONS/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/RESOURCEGROUPS/SA-HEMANT/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/AZURELSEVENTS/AUTHORIZATIONRULES/ROOTMANAGESHAREDACCESSKEY", - "level" : "Error", - "operationName" : "MICROSOFT.EVENTHUB/NAMESPACES/AUTHORIZATIONRULES/LISTKEYS/ACTION", - "fileset" : { - "name" : "activitylogs" - }, - "resultSignature" : "Failed.NotFound", - "input" : { - "type" : "azure" - }, - "@timestamp" : "2019-09-26T12:24:53.623Z", - "service" : { - "type" : "azure" + "category" : "Action", + "dataset" : "azure.activitylogs", + "outcome" : "Failure" }, - "kafka" : { - "headers" : [ - "ProfileName: �\fname=default" - ], - "partition" : 2, - "offset" : 12887, - "topic" : "insights-operational-logs", - "key" : "" - }, - "location" : "global", - "category" : "Action", - "durationMs" : 11, - "properties" : { - "serviceRequestId" : null, - "statusMessage" : """{"error":{"code":"ParentResourceNotFound","message":"Can not perform requested operation on nested resource. Parent resource 'azurelsevents' not found."}}""", - "statusCode" : "NotFound" + "azure" : { + "activitylogs" : { + "resourceID" : "/SUBSCRIPTIONS/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/RESOURCEGROUPS/SA-HEMANT/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/AZURELSEVENTS/AUTHORIZATIONRULES/ROOTMANAGESHAREDACCESSKEY", + "operationName" : "MICROSOFT.EVENTHUB/NAMESPACES/AUTHORIZATIONRULES/LISTKEYS/ACTION", + "resultSignature" : "Failed.NotFound", + "identity" : { + "authorization" : { + "evidence" : { + "roleAssignmentScope" : "/subscriptions/2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "roleDefinitionId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "role" : "Azure EventGrid Service BuiltIn Role", + "roleAssignmentId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "principalId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "principalType" : "ServicePrincipal" + }, + "scope" : "/subscriptions/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/resourceGroups/sa-hemant/providers/Microsoft.EventHub/namespaces/azurelsevents/authorizationRules/RootManageSharedAccessKey", + "action" : "Microsoft.EventHub/namespaces/authorizationRules/listKeys/action" + }, + "claims" : { + "http://schemas.xmlsoap.org/ws/2005/05/identity/claims/nameidentifier" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "ver" : "1.0", + "http://schemas.microsoft.com/identity/claims/identityprovider" : "https://sts.windows.net/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/", + "aio" : "2a7e2503-d7e2", + "iss" : "https://sts.windows.net/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/", + "uti" : "2a7e2503-d7e2", + "aud" : "https://management.core.windows.net/", + "nbf" : "1570547988", + "appidacr" : "2", + "http://schemas.microsoft.com/identity/claims/tenantid" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "appid" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "http://schemas.microsoft.com/identity/claims/objectidentifier" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "exp" : "1570577088", + "iat" : "1570547988" + } + }, + "correlationId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "properties" : { + "serviceRequestId" : null, + "statusMessage" : """{"error":{"code":"ParentResourceNotFound","message":"Can not perform requested operation on nested resource. Parent resource 'azurelsevents' not found."}}""", + "statusCode" : "NotFound" + } + } } }, "sort" : [ - 1569500693623 + 1570548289119 ] -} +}, diff --git a/x-pack/filebeat/module/azure/fields.go b/x-pack/filebeat/module/azure/fields.go index 2a2bdeb5ace..c406e630df7 100644 --- a/x-pack/filebeat/module/azure/fields.go +++ b/x-pack/filebeat/module/azure/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAzure returns asset data. // This is the base64 encoded gzipped contents of module/azure. func AssetAzure() string { - return "eJzsW81u2zgQvvcpBjkWaB8ghwVce3dhoG2K/PRaMOJE4YYmtSTlwH36BSmLUiJSYmLR7hbRqamU+T4OZ4Yfh8wHeMDdOZCftcJ3AIYZjudw5n4+ewdAUReKVYZJcQ5/vAOA5lv4ImnN7a/cMeRUn7tXH0CQDXbm7GN2FZ5DqWRd7f8nYPOpmSemCsO2zOy4LLV/GbIatdw8fzn7cCfVfgCLvWGwlj/2vn3OpM+GURSGmd2TlzE2E4zsc32PUBAhBSsIh1qjgvUK5B2YewT5KFC1P2hZqwLhti4e0Hx8ZihEuU+74IRt9Mf3gw9a6vL2HyxM4HXz4kfz1QPuHqWi45/92JCqYqLc/87Z+7PA9xNusc/ScY4OidTmXir2k1gb0XGFpiQR/6ukCIRzWRBDbjlCJemQTtz3fbK4tYFTYPCjKa6JfF/CeYp3n7uSHBdas1JsUJirQlaxYXRDiQfKCwdkn0vJEYhnAHqUQp/2Cu+YYBZjPcYlF2fq4WG9SiJ8fJIvnP6T+LE39wl+rBQTBasIPzbXby3wi1he746dTx1PE8JuGcbSbIpRApNwEemv+CMl/SDkxdC2XyOlUshJsFqMIU+gLjuzz+PCpxjqmptAIBwAe+lswsBoD9MJikGoHobZiJTIOGWFyjniK9nMONSL1qyDGfHwFSsFMZ0ynWfI1s06aLnLdTtyw3CowEbUV4LyeonqmhjIN8/RvxpK8ZoyM7sOd1ZfJMJ/o9i1+l9Ox6//5DsqPSyOsxHYBsz7+kgMllKFtz6vwv4sy7DVFtKgIAHNcchwncXxQnyqMnG6BagaJn8f9pX7Wp/YcftT+9ZmQvJIARt9EfvPew8rpitOdoHUn4mL70XQBmlYCYZeuUSic8mkxrYr0uYe3RxOuGoseGciNRLIfSZcliXST7srVFsW3HLPEzwlUrjdgY7ADEp3ROjPQKYTQKOCnmWal5G58AlEDF6z7Nlj2Aa1IZthF2ViEZuJSNS6X9CIKtG0WiPUmTm4bXXtILxGem3Hio6WPEjcmSbuSldjVQ+mQnhmMpHNu5/C+H59RhLRroD3RLWgVKGOdfdm5AKsgjGwllKtUfnewnEC50aj6lop0/GzkZTdMaSB3U6I4wwd2S97xLgGap/UfqzAx++E10duGn3FR9iOwqZXjkwU2yqSRlNyegI3XnAa4ecTWzDDiLEKJsfysBZ3Um0ataCwJIoyUTqRtweW6pUrBqliqTJbMi2iEKnJs1drKWUK8kTAXpamVC34lbIqiSWpqmN33RdVldJvfz7xx6Y5nPYJnWFX09z5dBPHSE2o1EUf8rg1VQDA/y6VolI3E7mEHJrSnJmYBeRnyj985rNSMPHWtn5rW8+E/da2foL+W56bDnb5SRfAXoW4DlnM25lvSuJhrfkTdBMLhXZjlLeZuGxAgBKDrqEYpWOlT/aTASdwkk4Ffp0OfIomnMs1E9qvTymXV26aW5tR7NimaI7Od3jz00POf3RVVWnxOaYe56gbFZCI/e4WLENhFlV1ozFXhjgI63gbckMjXbIK6m4GEr4oCtT6yhBTZ3LNsgMD4tBAh+G8cFOsZILwS/y3Rh2+9TfHwdUeBlSDMxbLTK+FQeVOlDKF8loDG8Hw8k4+oFhrXaPKl1bXFgSYQxnPqx6dfEeMT+iMnjJWStoIY6K0a/RafGGcM4024DOF9zeP6NZrYAI2Y6BePjL9sEJDGM/D65LpB6BhgD6Hz7hFvihLhaVVHhnJcIsEJA41oLWqFROl3Vqsc1016IhRB+bUaQCtz81Wy0yR7vjooP3nG/nsa6vf3SctsJGaPsOhQXRxSvrbB6WkWkp6hOPAPy0UFCEs3/zDLStwIu1P5aqGXLQ1PeeJu0Oa6kHvOyyivNppg5v8tC5aQNBxxJbdrZKPU03yWVh9igB1N46KbH+FdFA8FcNGxVNis7hnGUPpF6YjZH9sUeir7loYtbtQl1iG52tmSssGEKQCFYds6ZUol1IqygQx+W8m/J2ClnoMxIlhpo7WeMhzMvB5CrXLUFGegp+H/S8AAP//oxWmiQ==" + return "eJzsW91u2zgTve9TDHJZoH2AXHyAa3+7MNA2RX56WzDiROGGJrkk5cB9+gUli1IsUmJi0e4uqqumUuYckjPDM0PmAzzh7hLIz0rjOwDLLMdLuKh/vngHQNEUminLpLiE/70DgOZb+CJpxd2vPDDk1FzWrz6AIBvszLnH7hReQqllpfb/E7D50swLU4VlW2Z3XJbGvwxZjVpunj9q+/Ag9X4Ai71hcJY/9r49ZNJnwygKy+zuxcsYmwlG7rl9RCiIkIIVhENlUMN6BfIB7COCfBao2x+MrHSBcF8VT2g/HhgKUe7TLjhhG/Px/eCDlrq8/wsLG3jdvPjRfPWEu2ep6fhnPzZEKSbK/e9cvL8IfD8xLe5Z1pyjQyKVfZSa/STORnRcoSVJxP8qKQLhXBbEknuOoCQd0onPfZ8sbp3jFBj8aIprIt/XcJ7i3eeuJceFMawUGxT2ppAqNoxuKHFHeeWA3HMtOQLxDMCMUujTXuEDE8xhrMe45OJMPTysV0mET0/ylct/lnnsrX3CPCrNRMEU4afm+q0FfhXL292p46njaUPYLcNYmE0xSmASTiL9HX8kpR+FvBja9nuk1Bo5CWaLMeQJ1GVn9tAvfIhhs7kP3OYI2Ou9zRimVKhrUl/J5nApjoC9as3WMLHRVtzesFIQ26nEeYZccQsmaLmLOzdyy3CohkaUUIIKeo0CmhjIN8/RvxrK4ooyO7smrq2+ShD/h3zXaXE57b/+k++ozTBRzUZgGzDvcxWxWEodLkPehP1ZlmGrLaRFQQL7/zHDrS2OJMUzponzbQZqGPx92DfWmD6w4/anashmQfJsy877IvYP+wArZhQnu0Doz8TF9wVogzTMBMNZuUZickmWxnadpO0j1ms4MVVjzjsTqRFH7jPhsiyRftrdoN6yYPk7j/OUSOF+ByYCM0jdEdE9A5lOAI2Ka5ZpXUbWwgcQsXjLskePZRs0lmyGHY2JTWwmIlHrfkMjukTbao1Ql+ToFtJtDeE10lu7R3Q05UFilZhYIa7Gsh5MufDMZCKFtF/CeO08I4lohe5nQi0o1WhinbYZuQBTMAbWUqoMal/nn8Zx7gzqrq0x7T8bSdkDQxqodkIcZ+iOftkjxjVQ+6T2RgU+fye8OnED5ys+w3YUNj1zZKLYZpE0mpLTM0zjFacRfj6wBbOMWKdgcmwPa/Eg9aZRCxpLoikTZS3y9sBSv3HHICoWKrMF0yIKkRo8e7WWkqYgjwfsZWlK1oJfKaqSWBKlTt0BXyiV0vs+XPhT0xwu+4TOcLtp7ni6i2OkBlTqpg95pjVVAMC/LpSiUjcTuYQYmtKcmZgF5GfKP3zks1Iw8btt/bttPRP277b1C/SznmFW3AZK5eOHPKjyky5jvQlxHbKYtzPfpMTjWvNn6CYWGl1hlLeZuGxAgBKLdUMxSsdJn+wnA7XASToV+HU68CmacK6pmdB+fUq5ZuWuuUEZxY4VRXN0vsPFTw85/9GVUmn+OaYe58gbCkjEfncjlaGwC6XuDOaKkBrCTbxzuaGRLlgFrW/pEb4oCjTmxhJbZZqaZQcGpEYDE4bzwk2zkgnCr/HvCk34Bt4cB1d7GNANzpgvM7MWFnV9opTJldcG2AiGl3fyCcXamAp1vrC6dSDAapTxuOrRyXfE+ILO6Cmj0tJ5GBOl26PX4gvjnBl0Dp/Jvb95xHq/BiZgMwbq5SMzTyu0hPE8vK6ZeQIaBuhz+Ixb5Iuy1Fg65ZGRDHdIQOJQA1qrSjNRutJineuqQUeM1mC1Og2g9bm5bJnJ02s+Jmj/sJDPvrf66j5pg43k9BkODaKbU9LfIWgt9VLSExwH/t9BQRHC8s0/3LICJ8L+XFPVkIu2puc8ca+RpnrQ+w6LKG92xuImP62rFhBMHLFld6/l81STfBZWnyJA3Y2jIttfBB3lT8WwUfGS2CzTs4yh9BPTCaI/tin0VXclrN5d6Wssw+s1M6VlAwhSg45DtvRKlEspNWWC2Pw3E/5MQUs9BuLEMltFczzkORn4PIXaRagoz8HPw/4TAAD//2r4ijo=" } diff --git a/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.json b/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.json index 72c6123c81b..69a58f07ab9 100644 --- a/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.json +++ b/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.json @@ -46,9 +46,24 @@ "ignore_missing": true } }, + { + "rename": { + "field": "azure.signinlogs.resultType", + "target_field": "event.outcome", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.category", + "target_field": "event.category", + "ignore_missing": true + } + }, { "remove": { - "field": ["message", "azure.signinlogs.time","azure.signinlogs.Level", "azure.signinlogs.callerIpAddress","azure.signinlogs.durationMs"], + "field": ["message", "azure.signinlogs.time","azure.signinlogs.Level", "azure.signinlogs.callerIpAddress","azure.signinlogs.durationMs", + "azure.signinlogs.resultType", "azure.signinlogs.category"], "ignore_missing": true } }, From 9a2f77b19fe16f6c0ad4f9ed77f03b2ed1cc636b Mon Sep 17 00:00:00 2001 From: Mariana Date: Wed, 9 Oct 2019 11:11:07 +0200 Subject: [PATCH 16/21] Work on event format --- filebeat/input/kafka/config.go | 2 +- .../activitylogs/config/activitylogs.yml | 2 +- .../azure/activitylogs/ingest/pipeline.json | 12 +- .../test/activity_log_expected.json | 26 ++-- .../azure/auditlogs/config/auditlogs.yml | 2 +- .../azure/auditlogs/ingest/pipeline.json | 27 +++- .../auditlogs/test/audit_log_expected.json | 142 +++++++++--------- .../azure/signinlogs/config/signinlogs.yml | 2 +- .../azure/signinlogs/ingest/pipeline.json | 14 +- .../signinlogs/test/signin_log_expected.json | 136 +++++++++++++++++ 10 files changed, 272 insertions(+), 93 deletions(-) diff --git a/filebeat/input/kafka/config.go b/filebeat/input/kafka/config.go index e563248bddf..e78e58536be 100644 --- a/filebeat/input/kafka/config.go +++ b/filebeat/input/kafka/config.go @@ -50,7 +50,7 @@ type kafkaInputConfig struct { TLS *tlscommon.Config `config:"ssl"` Username string `config:"username"` Password string `config:"password"` - YieldEventsFromField string `config:"yield_events_from_field"` + YieldEventsFromField string `config:"expand_event_list_from_field"` } type kafkaFetch struct { diff --git a/x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml b/x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml index edcdfb94723..155ab319668 100644 --- a/x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml +++ b/x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml @@ -2,7 +2,7 @@ type: kafka hosts: {{ .namespace }} topics: {{ .eventhub }} group_id: {{ .consumer_group }} -yield_events_from_field: "records" +expand_event_list_from_field: "records" username: "$ConnectionString" password: {{ .connection_string }} diff --git a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json index 9ac260d8cb3..b2da0fcb1f1 100644 --- a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json +++ b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json @@ -39,6 +39,15 @@ "ignore_missing": true } }, + { + "script": { + "lang": "painless", + "source": "ctx.event.duration = ctx.event.duration * params.param_nano", + "params": { + "param_nano": 1000000 + } + } + }, { "rename": { "field": "azure.activitylogs.location", @@ -62,8 +71,7 @@ }, { "remove": { - "field": ["message", "azure.activitylogs.time", "azure.activitylogs.callerIpAddress", "azure.activitylogs.category", - "azure.activitylogs.level", "azure.activitylogs.durationMs", "azure.activitylogs.location", "azure.activitylogs.resultType"], + "field": ["message", "azure.activitylogs.time"], "ignore_missing": true } }, diff --git a/x-pack/filebeat/module/azure/activitylogs/test/activity_log_expected.json b/x-pack/filebeat/module/azure/activitylogs/test/activity_log_expected.json index c32ebc37934..2dcea76b480 100644 --- a/x-pack/filebeat/module/azure/activitylogs/test/activity_log_expected.json +++ b/x-pack/filebeat/module/azure/activitylogs/test/activity_log_expected.json @@ -1,7 +1,7 @@ { - "_index" : "filebeat-8.0.0-2019.10.08", + "_index" : "filebeat-8.0.0-2019.10.09", "_type" : "_doc", - "_id" : "2a7e2503-d7e2", + "_id" : "2a7e2503-", "_score" : null, "_source" : { "agent" : { @@ -15,7 +15,7 @@ "level" : "Error" }, "source" : { - "ip" : "51.161.101.191" + "ip" : "51.221.31.31" }, "fileset" : { "name" : "activitylogs" @@ -26,7 +26,7 @@ "input" : { "type" : "kafka" }, - "@timestamp" : "2019-10-08T15:24:49.119Z", + "@timestamp" : "2019-10-09T08:24:34.373Z", "ecs" : { "version" : "1.1.0" }, @@ -38,7 +38,7 @@ "ProfileName: �\fname=default" ], "partition" : 3, - "offset" : 44768, + "offset" : 46716, "topic" : "insights-operational-logs", "key" : "" }, @@ -57,7 +57,7 @@ "architecture" : "x86_64" }, "event" : { - "duration" : 11, + "duration" : 1.1E7, "module" : "azure", "category" : "Action", "dataset" : "azure.activitylogs", @@ -85,17 +85,17 @@ "http://schemas.xmlsoap.org/ws/2005/05/identity/claims/nameidentifier" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", "ver" : "1.0", "http://schemas.microsoft.com/identity/claims/identityprovider" : "https://sts.windows.net/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/", - "aio" : "2a7e2503-d7e2", + "aio" : "2a7e2503-d7e2-405a-a84c-c3", "iss" : "https://sts.windows.net/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/", - "uti" : "2a7e2503-d7e2", + "uti" : "7m-OZFTlhUSg2aXLucMuAA", "aud" : "https://management.core.windows.net/", - "nbf" : "1570547988", + "nbf" : "1570609174", "appidacr" : "2", "http://schemas.microsoft.com/identity/claims/tenantid" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", "appid" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", "http://schemas.microsoft.com/identity/claims/objectidentifier" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", - "exp" : "1570577088", - "iat" : "1570547988" + "exp" : "1570638274", + "iat" : "1570609174" } }, "correlationId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", @@ -108,6 +108,6 @@ } }, "sort" : [ - 1570548289119 + 1570609474373 ] -}, +} diff --git a/x-pack/filebeat/module/azure/auditlogs/config/auditlogs.yml b/x-pack/filebeat/module/azure/auditlogs/config/auditlogs.yml index 4221a297987..b36bbeacae6 100644 --- a/x-pack/filebeat/module/azure/auditlogs/config/auditlogs.yml +++ b/x-pack/filebeat/module/azure/auditlogs/config/auditlogs.yml @@ -2,7 +2,7 @@ type: kafka hosts: {{ .namespace }} topics: {{ .eventhub }} group_id: {{ .consumer_group }} -yield_events_from_field: "records" +expand_event_list_from_field: "records" username: "$ConnectionString" password: {{ .connection_string }} diff --git a/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.json b/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.json index 33badd8ee2d..4ab40fd5555 100644 --- a/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.json +++ b/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.json @@ -25,6 +25,22 @@ "ignore_missing": true } }, + { + "script": { + "lang": "painless", + "source": "ctx.event.duration = ctx.event.duration * params.param_nano", + "params": { + "param_nano": 1000000 + } + } + }, + { + "rename": { + "field": "azure.auditlogs.properties.result", + "target_field": "event.outcome", + "ignore_missing": true + } + }, { "rename": { "field": "azure.auditlogs.level", @@ -34,13 +50,20 @@ }, { "remove": { - "field": ["message", "azure.auditlogs.time", "azure.auditlogs.durationMs", "azure.auditlogs.level"], + "field": ["message", "azure.auditlogs.time"], + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.auditlogs.category", + "target_field": "event.category", "ignore_missing": true } }, { "drop": { - "if" : "ctx.azure.auditlogs.category != 'AuditLogs'" + "if" : "ctx.event.category != 'AuditLogs'" } } ], diff --git a/x-pack/filebeat/module/azure/auditlogs/test/audit_log_expected.json b/x-pack/filebeat/module/azure/auditlogs/test/audit_log_expected.json index 3d3a9d95e22..ec57eda58c4 100644 --- a/x-pack/filebeat/module/azure/auditlogs/test/audit_log_expected.json +++ b/x-pack/filebeat/module/azure/auditlogs/test/audit_log_expected.json @@ -1,7 +1,7 @@ { - "_index" : "filebeat-8.0.0-2019.10.04", + "_index" : "filebeat-8.0.0-2019.10.09", "_type" : "_doc", - "_id" : "FXFblm0Bq15PGrhqyTlD", + "_id" : "2a7e2503-d7e2-405", "_score" : null, "_source" : { "agent" : { @@ -11,14 +11,34 @@ "ephemeral_id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", "version" : "8.0.0" }, + "log" : { + "level" : "Informational" + }, + "fileset" : { + "name" : "auditlogs" + }, + "input" : { + "type" : "kafka" + }, + "@timestamp" : "2019-10-08T22:18:17.146Z", "ecs" : { "version" : "1.1.0" }, + "service" : { + "type" : "azure" + }, + "kafka" : { + "headers" : [ ], + "partition" : 3, + "offset" : 28, + "topic" : "insights-logs-auditlogs", + "key" : "" + }, "host" : { "hostname" : "DESKTOP-RFOOE09", "os" : { - "build" : "18362.387", - "kernel" : "10.0.18362.387 (WinBuild.160101.0800)", + "build" : "18362.388", + "kernel" : "10.0.18362.388 (WinBuild.160101.0800)", "name" : "Windows 10 Pro", "family" : "windows", "version" : "10.0", @@ -28,80 +48,64 @@ "id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", "architecture" : "x86_64" }, - "correlationId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", "event" : { + "duration" : 0.0, "module" : "azure", - "dataset" : "azure.auditlogs" + "category" : "AuditLogs", + "dataset" : "azure.auditlogs", + "outcome" : "success" }, - "resourceID" : "/tenants/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/providers/Microsoft.aadiam", - "level" : "Informational", - "specification" : "Device Registration Service", - "operationName" : "Update device", - "fileset" : { - "name" : "auditlogs" - }, - "resultSignature" : "None", - "input" : { - "type" : "kafka" - }, - "@timestamp" : "2019-10-04T10:05:32.944Z", - "operationVersion" : "1.0", - "service" : { - "type" : "azure" - }, - "kafka" : { - "headers" : [ ], - "partition" : 3, - "offset" : 2, - "topic" : "insights-logs-auditlogs", - "key" : "" - }, - "tenantId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", - "category" : "AuditLogs", - "durationMs" : 0, - "properties" : { - "result" : "success", - "activityDisplayName" : "Update device", - "resultReason" : "", - "correlationId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", - "loggedByService" : "Core Directory", - "operationType" : "Update", - "id" : "Directory_2a7e2503", - "activityDateTime" : "2019-10-04T10:05:32.9448766+00:00", - "additionalDetails" : [ ], - "targetResources" : [ - { - "displayName" : "win10odbc", - "modifiedProperties" : [ + "azure" : { + "auditlogs" : { + "resourceID" : "/tenants/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/providers/Microsoft.aadiam", + "operationVersion" : "1.0", + "tenantId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "operationName" : "Disable account", + "correlationId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "properties" : { + "activityDisplayName" : "Disable account", + "resultReason" : "", + "operationType" : "Update", + "correlationId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "loggedByService" : "Core Directory", + "id" : "Directory_U6", + "activityDateTime" : "2019-10-08T22:18:17.1467655+00:00", + "additionalDetails" : [ ], + "category" : "UserManagement", + "targetResources" : [ { - "newValue" : "", - "displayName" : "Included Updated Properties", - "oldValue" : "" + "displayName" : null, + "modifiedProperties" : [ + { + "newValue" : "[false]", + "displayName" : "AccountEnabled", + "oldValue" : "[true]" + }, + { + "newValue" : """"AccountEnabled"""", + "displayName" : "Included Updated Properties", + "oldValue" : null + } + ], + "id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "type" : "User", + "userPrincipalName" : "test.test@elastic.co" } ], - "id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", - "type" : "Device", - "userPrincipalName" : "" - } - ], - "category" : "Device", - "initiatedBy" : { - "app" : { - "servicePrincipalName" : null, - "displayName" : "Device Registration Service", - "appId" : null, - "servicePrincipalId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73" + "initiatedBy" : { + "user" : { + "displayName" : null, + "ipAddress" : null, + "id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "userPrincipalName" : "admintestsds@elastic365.onmicrosoft.com" + } + } }, - "user" : { - "displayName" : null, - "ipAddress" : null, - "id" : "", - "userPrincipalName" : "" - } + "resultSignature" : "None" } } }, "sort" : [ - 1570183532944 + 1570573097146 ] -} +}, diff --git a/x-pack/filebeat/module/azure/signinlogs/config/signinlogs.yml b/x-pack/filebeat/module/azure/signinlogs/config/signinlogs.yml index a87b87bf7bb..0b8c0b3237b 100644 --- a/x-pack/filebeat/module/azure/signinlogs/config/signinlogs.yml +++ b/x-pack/filebeat/module/azure/signinlogs/config/signinlogs.yml @@ -2,7 +2,7 @@ type: kafka hosts: {{ .namespace }} topics: {{ .eventhub }} group_id: {{ .consumer_group }} -yield_events_from_field: "records" +expand_event_list_from_field: "records" username: "$ConnectionString" password: {{ .connection_string }} diff --git a/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.json b/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.json index 69a58f07ab9..e55c86a64a8 100644 --- a/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.json +++ b/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.json @@ -39,6 +39,15 @@ "ignore_missing": true } }, + { + "script": { + "lang": "painless", + "source": "ctx.event.duration = ctx.event.duration * params.param_nano", + "params": { + "param_nano": 1000000 + } + } + }, { "rename": { "field": "azure.signinlogs.location", @@ -62,14 +71,13 @@ }, { "remove": { - "field": ["message", "azure.signinlogs.time","azure.signinlogs.Level", "azure.signinlogs.callerIpAddress","azure.signinlogs.durationMs", - "azure.signinlogs.resultType", "azure.signinlogs.category"], + "field": ["message", "azure.signinlogs.time"], "ignore_missing": true } }, { "drop": { - "if" : "ctx.azure.signinlogs.category != 'SignInLogs'" + "if" : "ctx.event.category != 'SignInLogs'" } } ], diff --git a/x-pack/filebeat/module/azure/signinlogs/test/signin_log_expected.json b/x-pack/filebeat/module/azure/signinlogs/test/signin_log_expected.json index e69de29bb2d..975095d8909 100644 --- a/x-pack/filebeat/module/azure/signinlogs/test/signin_log_expected.json +++ b/x-pack/filebeat/module/azure/signinlogs/test/signin_log_expected.json @@ -0,0 +1,136 @@ +{ + "_index" : "filebeat-8.0.0-2019.10.08", + "_type" : "_doc", + "_id" : "2a7e2503-d7e2-405a", + "_score" : null, + "_source" : { + "agent" : { + "hostname" : "DESKTOP-RFOOE09", + "id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "type" : "filebeat", + "ephemeral_id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "version" : "8.0.0" + }, + "log" : { + "level" : 4 + }, + "source" : { + "ip" : "71.251.171.21" + }, + "fileset" : { + "name" : "signinlogs" + }, + "geo" : { + "country_iso_code" : "US" + }, + "input" : { + "type" : "kafka" + }, + "@timestamp" : "2019-10-08T21:32:19.991Z", + "ecs" : { + "version" : "1.1.0" + }, + "service" : { + "type" : "azure" + }, + "kafka" : { + "headers" : [ ], + "partition" : 0, + "offset" : 700, + "topic" : "insights-logs-signinlogs", + "key" : "" + }, + "host" : { + "hostname" : "DESKTOP-RFOOE09", + "os" : { + "build" : "18362.388", + "kernel" : "10.0.183.388 (WinBuild.160101.0800)", + "name" : "Windows 10 Pro", + "family" : "windows", + "version" : "10.0", + "platform" : "windows" + }, + "name" : "DESKTOP-RFOOE09", + "id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "architecture" : "x86_64" + }, + "event" : { + "duration" : 0.0, + "module" : "azure", + "category" : "SignInLogs", + "dataset" : "azure.signinlogs", + "outcome" : "0" + }, + "azure" : { + "signinlogs" : { + "resourceID" : "/tenants/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/providers/Microsoft.aadiam", + "operationName" : "Sign-in activity", + "resultSignature" : "None", + "operationVersion" : "1.0", + "identity" : "Test Test", + "tenantId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "correlationId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "properties" : { + "networkLocationDetails" : [ ], + "resourceId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "appliedConditionalAccessPolicies" : [ ], + "deviceDetail" : { + "displayName" : "DESKTOP-MJBCS48", + "trustType" : "Azure AD joined", + "deviceId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "operatingSystem" : "Windows" + }, + "isInteractive" : false, + "createdDateTime" : "2019-10-08T21:32:19.9914118+00:00", + "clientAppUsed" : "Mobile Apps and Desktop clients", + "riskLevelAggregated" : "none", + "authenticationProcessingDetails" : [ ], + "processingTimeInMilliseconds" : 96, + "appId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "correlationId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "riskState" : "none", + "authenticationDetails" : [ + { + "authenticationStepDateTime" : "2019-10-08T21:32:19.9914118+00:00", + "authenticationStepResultDetail" : "None", + "authenticationStepRequirement" : "Multi Factor Authentication", + "succeeded" : false + } + ], + "userPrincipalName" : "test.test@elastic.co", + "servicePrincipalId" : "", + "tokenIssuerType" : "AzureAD", + "resourceDisplayName" : "Windows Azure Active Directory", + "originalRequestId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "riskEventTypes" : [ ], + "appDisplayName" : "Windows Sign In", + "riskLevelDuringSignIn" : "none", + "ipAddress" : "71.251.171.21", + "authenticationRequirementPolicies" : [ ], + "userDisplayName" : "Test Test", + "userId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "tokenIssuerName" : "", + "riskDetail" : "none", + "location" : { + "countryOrRegion" : "US", + "city" : "San Leandro", + "geoCoordinates" : { + "latitude" : 37.3444379492, + "longitude" : -123.2354456456 + }, + "state" : "California" + }, + "status" : { + "errorCode" : 0, + "additionalDetails" : "MFA claim has expired due to the policies configured on tenant" + }, + "mfaDetail" : { } + } + } + } + }, + "sort" : [ + 1570570339991 + ] +} From f18c856c21f363eb37e950551c91b07440357bd3 Mon Sep 17 00:00:00 2001 From: Mariana Date: Wed, 9 Oct 2019 15:38:42 +0200 Subject: [PATCH 17/21] Work on filesets --- CHANGELOG.next.asciidoc | 2 +- filebeat/docs/fields.asciidoc | 10 +++++----- filebeat/docs/modules/azure.asciidoc | 7 +++++++ filebeat/filebeat.yml | 4 +++- filebeat/input/kafka/input.go | 3 ++- x-pack/filebeat/filebeat.reference.yml | 4 ++++ x-pack/filebeat/module/azure/_meta/config.yml | 4 ++++ x-pack/filebeat/module/azure/_meta/docs.asciidoc | 7 +++++++ x-pack/filebeat/module/azure/_meta/fields.yml | 4 ++-- .../module/azure/activitylogs/_meta/fields.yml | 2 +- .../module/azure/activitylogs/config/activitylogs.yml | 5 ----- .../module/azure/activitylogs/ingest/pipeline.json | 6 ++++++ .../filebeat/module/azure/auditlogs/_meta/fields.yml | 2 +- .../module/azure/auditlogs/config/auditlogs.yml | 5 ----- .../module/azure/auditlogs/ingest/pipeline.json | 6 ++++++ x-pack/filebeat/module/azure/fields.go | 2 +- x-pack/filebeat/module/azure/module.yml | 4 +--- .../filebeat/module/azure/signinlogs/_meta/fields.yml | 2 +- .../module/azure/signinlogs/config/signinlogs.yml | 5 ----- .../module/azure/signinlogs/ingest/pipeline.json | 6 ++++++ .../module/cisco/shared/gen-ecs-mapping-docs.go | 4 +--- x-pack/filebeat/modules.d/azure.yml.disabled | 4 ++++ 22 files changed, 63 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 0a7bc712c20..436fe94a928 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -329,7 +329,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add module for ingesting Cisco FTD logs over syslog. {pull}13286[13286] - Update CoreDNS module to populate ECS DNS fields. {issue}13320[13320] {pull}13505[13505] - Parse query steps in PostgreSQL slowlogs. {issue}13496[13496] {pull}13701[13701] -- Add azure module. {pull}13776[13776] +- Add filebeat azure module with activitylogs, auditlogs, signinlogs filesets. {pull}13776[13776] - Add support to set the document id in the json reader. {pull}5844[5844] - Add input httpjson. {issue}13545[13545] {pull}13546[13546] - Filebeat Netflow input: Remove beta label. {pull}13858[13858] diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index f5c748b19d1..8f6d259e2ce 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -1243,9 +1243,9 @@ type: keyword -- [[exported-fields-azure]] -== azure fields +== Azure fields -azure Module +Azure Module @@ -1258,7 +1258,7 @@ azure Module [float] === activitylogs -Fields for azure Activity logs. +Fields for Azure activity logs. @@ -1426,7 +1426,7 @@ type: object [float] === auditlogs -Fields for azure audit logs. +Fields for Azure audit logs. @@ -1795,7 +1795,7 @@ type: keyword [float] === signinlogs -Fields for azure audit logs. +Fields for Azure sign-in logs. diff --git a/filebeat/docs/modules/azure.asciidoc b/filebeat/docs/modules/azure.asciidoc index d428fb6f382..769196265ca 100644 --- a/filebeat/docs/modules/azure.asciidoc +++ b/filebeat/docs/modules/azure.asciidoc @@ -15,6 +15,13 @@ beta[] This is the azure module. The azure module will concentrate on retrieving different types of log data from Azure. +There are several requirements before using the module since the logs will actually be read from azure event hubs. + + - the event hubs the azure module will read logs from must have the kafka option enabled . + - the logs have to be exported first to the event hubs https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-create-kafka-enabled + - to export activity logs to event hubs users can follow the steps here https://docs.microsoft.com/en-us/azure/azure-monitor/platform/activity-log-export + - to export audit and sign-in logs to event hubs users can follow the steps here https://docs.microsoft.com/en-us/azure/active-directory/reports-monitoring/tutorial-azure-monitor-stream-logs-to-event-hub + The module will contain the following filesets: `activitylogs` :: diff --git a/filebeat/filebeat.yml b/filebeat/filebeat.yml index d02b4d161fb..067af19c22d 100644 --- a/filebeat/filebeat.yml +++ b/filebeat/filebeat.yml @@ -11,7 +11,9 @@ # configuration file. #=========================== Filebeat inputs ============================= - +logging.level: debug +filebeat.overwrite_pipelines: true +setup.ilm.enabled: false filebeat.inputs: # Each - is an input. Most options can be set at the input level, so diff --git a/filebeat/input/kafka/input.go b/filebeat/input/kafka/input.go index 6feb5cfd742..809190830cd 100644 --- a/filebeat/input/kafka/input.go +++ b/filebeat/input/kafka/input.go @@ -348,8 +348,9 @@ func (h *groupHandler) parseMultipleMessages(bMessage []byte) []string { for _, ms := range obj[h.expandEventListFromField] { js, err := json.Marshal(ms) if err == nil { - h.log.Errorw(fmt.Sprintf("Kafka serializing message %s", ms), "error", err) messages = append(messages, string(js)) + }else { + h.log.Errorw(fmt.Sprintf("Kafka serializing message %s", ms), "error", err) } } } diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 65aa179eb79..979d7b66972 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -93,9 +93,13 @@ filebeat.modules: activitylogs: enabled: true var: + # Azure event hub namespace FQDN for example "eventhubs.servicebus.windows.net:9093" namespace: "" + # Eventhub name containing the activity logs, overwrite he default value if the logs are exported in a different eventhub eventhub: ["insights-operational-logs"] + # Consumer group name that has access to the event hub, we advise creating a dedicated consumer group for the azure module consumer_group: "$Default" + # the connection string required to communicate with Event Hubs, steps to generate one here https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string connection_string: "" # auditlogs: diff --git a/x-pack/filebeat/module/azure/_meta/config.yml b/x-pack/filebeat/module/azure/_meta/config.yml index d8b7689d285..baaccbfd3b8 100644 --- a/x-pack/filebeat/module/azure/_meta/config.yml +++ b/x-pack/filebeat/module/azure/_meta/config.yml @@ -3,9 +3,13 @@ activitylogs: enabled: true var: + # Azure event hub namespace FQDN for example "eventhubs.servicebus.windows.net:9093" namespace: "" + # Eventhub name containing the activity logs, overwrite he default value if the logs are exported in a different eventhub eventhub: ["insights-operational-logs"] + # Consumer group name that has access to the event hub, we advise creating a dedicated consumer group for the azure module consumer_group: "$Default" + # the connection string required to communicate with Event Hubs, steps to generate one here https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string connection_string: "" # auditlogs: diff --git a/x-pack/filebeat/module/azure/_meta/docs.asciidoc b/x-pack/filebeat/module/azure/_meta/docs.asciidoc index 33d84149d08..68013c1f248 100644 --- a/x-pack/filebeat/module/azure/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/azure/_meta/docs.asciidoc @@ -10,6 +10,13 @@ beta[] This is the azure module. The azure module will concentrate on retrieving different types of log data from Azure. +There are several requirements before using the module since the logs will actually be read from azure event hubs. + + - the event hubs the azure module will read logs from must have the kafka option enabled . + - the logs have to be exported first to the event hubs https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-create-kafka-enabled + - to export activity logs to event hubs users can follow the steps here https://docs.microsoft.com/en-us/azure/azure-monitor/platform/activity-log-export + - to export audit and sign-in logs to event hubs users can follow the steps here https://docs.microsoft.com/en-us/azure/active-directory/reports-monitoring/tutorial-azure-monitor-stream-logs-to-event-hub + The module will contain the following filesets: `activitylogs` :: diff --git a/x-pack/filebeat/module/azure/_meta/fields.yml b/x-pack/filebeat/module/azure/_meta/fields.yml index dc7c1477832..9b2efffbd48 100644 --- a/x-pack/filebeat/module/azure/_meta/fields.yml +++ b/x-pack/filebeat/module/azure/_meta/fields.yml @@ -1,7 +1,7 @@ - key: azure - title: "azure" + title: "Azure" description: > - azure Module + Azure Module fields: - name: azure type: group diff --git a/x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml b/x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml index ed5cd065446..d8db6b53e8c 100644 --- a/x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml +++ b/x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml @@ -1,7 +1,7 @@ - name: activitylogs type: group description: > - Fields for azure Activity logs. + Fields for Azure activity logs. fields: - name: identity type: group diff --git a/x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml b/x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml index 155ab319668..c22e189eb94 100644 --- a/x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml +++ b/x-pack/filebeat/module/azure/activitylogs/config/activitylogs.yml @@ -7,8 +7,3 @@ expand_event_list_from_field: "records" username: "$ConnectionString" password: {{ .connection_string }} ssl.enabled: true - -processors: -- decode_json_fields: - fields: ["message"] - target: "azure.activitylogs" diff --git a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json index b2da0fcb1f1..541a46f6802 100644 --- a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json +++ b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json @@ -1,6 +1,12 @@ { "description": "Pipeline for parsing azure activity logs.", "processors": [ + { + "json" : { + "field" : "message", + "target_field" : "azure.activitylogs" + } + }, { "date": { "field": "azure.activitylogs.time", diff --git a/x-pack/filebeat/module/azure/auditlogs/_meta/fields.yml b/x-pack/filebeat/module/azure/auditlogs/_meta/fields.yml index c76e45b11a6..f792cfcd9e5 100644 --- a/x-pack/filebeat/module/azure/auditlogs/_meta/fields.yml +++ b/x-pack/filebeat/module/azure/auditlogs/_meta/fields.yml @@ -1,7 +1,7 @@ - name: auditlogs type: group description: > - Fields for azure audit logs. + Fields for Azure audit logs. fields: - name: resourceID type: keyword diff --git a/x-pack/filebeat/module/azure/auditlogs/config/auditlogs.yml b/x-pack/filebeat/module/azure/auditlogs/config/auditlogs.yml index b36bbeacae6..c22e189eb94 100644 --- a/x-pack/filebeat/module/azure/auditlogs/config/auditlogs.yml +++ b/x-pack/filebeat/module/azure/auditlogs/config/auditlogs.yml @@ -7,8 +7,3 @@ expand_event_list_from_field: "records" username: "$ConnectionString" password: {{ .connection_string }} ssl.enabled: true - -processors: -- decode_json_fields: - fields: ["message"] - target: "azure.auditlogs" diff --git a/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.json b/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.json index 4ab40fd5555..fad8ddb8e43 100644 --- a/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.json +++ b/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.json @@ -1,6 +1,12 @@ { "description": "Pipeline for parsing azure activity logs.", "processors": [ + { + "json" : { + "field" : "message", + "target_field" : "azure.auditlogs" + } + }, { "date": { "field": "azure.auditlogs.time", diff --git a/x-pack/filebeat/module/azure/fields.go b/x-pack/filebeat/module/azure/fields.go index c406e630df7..0a74d94369c 100644 --- a/x-pack/filebeat/module/azure/fields.go +++ b/x-pack/filebeat/module/azure/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAzure returns asset data. // This is the base64 encoded gzipped contents of module/azure. func AssetAzure() string { - return "eJzsW91u2zgTve9TDHJZoH2AXHyAa3+7MNA2RX56WzDiROGGJrkk5cB9+gUli1IsUmJi0e4uqqumUuYckjPDM0PmAzzh7hLIz0rjOwDLLMdLuKh/vngHQNEUminLpLiE/70DgOZb+CJpxd2vPDDk1FzWrz6AIBvszLnH7hReQqllpfb/E7D50swLU4VlW2Z3XJbGvwxZjVpunj9q+/Ag9X4Ai71hcJY/9r49ZNJnwygKy+zuxcsYmwlG7rl9RCiIkIIVhENlUMN6BfIB7COCfBao2x+MrHSBcF8VT2g/HhgKUe7TLjhhG/Px/eCDlrq8/wsLG3jdvPjRfPWEu2ep6fhnPzZEKSbK/e9cvL8IfD8xLe5Z1pyjQyKVfZSa/STORnRcoSVJxP8qKQLhXBbEknuOoCQd0onPfZ8sbp3jFBj8aIprIt/XcJ7i3eeuJceFMawUGxT2ppAqNoxuKHFHeeWA3HMtOQLxDMCMUujTXuEDE8xhrMe45OJMPTysV0mET0/ylct/lnnsrX3CPCrNRMEU4afm+q0FfhXL292p46njaUPYLcNYmE0xSmASTiL9HX8kpR+FvBja9nuk1Bo5CWaLMeQJ1GVn9tAvfIhhs7kP3OYI2Ou9zRimVKhrUl/J5nApjoC9as3WMLHRVtzesFIQ26nEeYZccQsmaLmLOzdyy3CohkaUUIIKeo0CmhjIN8/RvxrK4ooyO7smrq2+ShD/h3zXaXE57b/+k++ozTBRzUZgGzDvcxWxWEodLkPehP1ZlmGrLaRFQQL7/zHDrS2OJMUzponzbQZqGPx92DfWmD6w4/anashmQfJsy877IvYP+wArZhQnu0Doz8TF9wVogzTMBMNZuUZickmWxnadpO0j1ms4MVVjzjsTqRFH7jPhsiyRftrdoN6yYPk7j/OUSOF+ByYCM0jdEdE9A5lOAI2Ka5ZpXUbWwgcQsXjLskePZRs0lmyGHY2JTWwmIlHrfkMjukTbao1Ql+ToFtJtDeE10lu7R3Q05UFilZhYIa7Gsh5MufDMZCKFtF/CeO08I4lohe5nQi0o1WhinbYZuQBTMAbWUqoMal/nn8Zx7gzqrq0x7T8bSdkDQxqodkIcZ+iOftkjxjVQ+6T2RgU+fye8OnED5ys+w3YUNj1zZKLYZpE0mpLTM0zjFacRfj6wBbOMWKdgcmwPa/Eg9aZRCxpLoikTZS3y9sBSv3HHICoWKrMF0yIKkRo8e7WWkqYgjwfsZWlK1oJfKaqSWBKlTt0BXyiV0vs+XPhT0xwu+4TOcLtp7ni6i2OkBlTqpg95pjVVAMC/LpSiUjcTuYQYmtKcmZgF5GfKP3zks1Iw8btt/bttPRP277b1C/SznmFW3AZK5eOHPKjyky5jvQlxHbKYtzPfpMTjWvNn6CYWGl1hlLeZuGxAgBKLdUMxSsdJn+wnA7XASToV+HU68CmacK6pmdB+fUq5ZuWuuUEZxY4VRXN0vsPFTw85/9GVUmn+OaYe58gbCkjEfncjlaGwC6XuDOaKkBrCTbxzuaGRLlgFrW/pEb4oCjTmxhJbZZqaZQcGpEYDE4bzwk2zkgnCr/HvCk34Bt4cB1d7GNANzpgvM7MWFnV9opTJldcG2AiGl3fyCcXamAp1vrC6dSDAapTxuOrRyXfE+ILO6Cmj0tJ5GBOl26PX4gvjnBl0Dp/Jvb95xHq/BiZgMwbq5SMzTyu0hPE8vK6ZeQIaBuhz+Ixb5Iuy1Fg65ZGRDHdIQOJQA1qrSjNRutJineuqQUeM1mC1Og2g9bm5bJnJ02s+Jmj/sJDPvrf66j5pg43k9BkODaKbU9LfIWgt9VLSExwH/t9BQRHC8s0/3LICJ8L+XFPVkIu2puc8ca+RpnrQ+w6LKG92xuImP62rFhBMHLFld6/l81STfBZWnyJA3Y2jIttfBB3lT8WwUfGS2CzTs4yh9BPTCaI/tin0VXclrN5d6Wssw+s1M6VlAwhSg45DtvRKlEspNWWC2Pw3E/5MQUs9BuLEMltFczzkORn4PIXaRagoz8HPw/4TAAD//2r4ijo=" + return "eJzsW91u2zgTve9TDHJZoH2AXHyAa3+7MNA2RX56WzDiRJkNTWpJyoH79AtK1k8sUmJi0e4uqqumUuYckjPDM0PmAzzh7hLYz1LjOwBLVuAlXCzczxfvADiaTFNhSclL+N87AIDqHXxRvBTuVx4IBTeX1asPINkGO3PusbsCLyHXqiz2/+Ox+dLMC1OZpS3ZnVC5aV/6rAYt188flX14UHo/gMYwOMsfe98eMumzIY7Skt29eBliM8HIPbePCBmTSlLGBJQGNaxXoB7APiKoZ4m6+cGoUmcI92X2hPbjgSEf5T7tTDDamI/vBx801NX9X5hZz+v6xY/6qyfcPSvNxz/7sWFFQTLf/87F+wvP9xPT4p5lxTk4JFbaR6XpJ3M2guPyLUkk/lfFEZgQKmOW3QuEQvEhnfDc98ni1jlOht6PprhG8n0N5ynefe5aCVwYQ7ncoLQ3mSpCw+iGEnaUVw7IPddKILCWAZhRCn3aK3wgSQ5jPcYlFWfewsN6FUX49CRfufxnmcfe2kfMY6FJZlQwcWqu3xrgV7G83Z06njqe1ofdMAyF2RSjCCb+JNLf8UdS+lHIi6Htdo9UWqNg3mwxhjyBuuzMHvpFG2JYb+4DtzkC9npvM4SpCtQVqa9sc7gUR8BeNWYrmNBoS2FvKJfMdipxniGXwoLxWu7izo3cEg7V0IgSilBBr1FAEwP51nJsXw1lccnJzq+JndVXCeL/kO86La6m/bf95DtqM0xUsxHYesy3uYpZzJX2lyFvwv6scr/VBtKiZJ79/5jhVhZHkuIZ08T5NoNiGPx92DfWmG1gh+1P1ZD1gqTZlp33Bewf9gFWZArBdp7Qn4nLoukL8BppmAmGs3KNzKSSLLXtKknbR6zWcGKqxpx3JlIjjtxnIlSeI/+0u0G9JW/5O4/z5MjhfgcmADNI3QHRPQOZTgCNimtKtC4ja9EGELN4S8mjx9IGjWWbYUdjYhObiUjQeruhMZ2jbbSGr0tydAvptoJoNdJbu0d8NOVBZJUYWSGuxrIeTLnwzGQChXS7hOHaeUYSwQq9nYliwblGE+q0zcgFqIAxsIZSaVC3df5pHOfOoO7aGtP+s1GcHgi5p9rxcZyhO/pljxjWQM0T2xuV+PydifLEDZyv+AzbUdj4zJGIYpNF4mgqwc8wjVeCB/i1gS3JErNOwaTYHtbyQelNrRY05kxzknkl8vbASr9xx2BFKFRmC6ZFECI2ePZqLSZNQRoP2MvSmKwFv1JURbFkRXHqDviiKGJ634cLf2qaw2Wf0BluN00dT3dhjNiAit30Ic20xgoA+NeFUlDqJiIXEUNTmjMRM4/8jPlHG/mUS5Kzt62d2Q8kfzeufzeuh5C/G9czYMedYpbCeorl44c8qPOjrmO9CXHts5i2N18nxeOa82foJ2YaXWmUtp24rEGAM4tVSzFIx4mf5GcDlcSJOhf4dXrwMapwrqmZUH99Sqlm5a6+QxnEDpVFc/S+/eVPDzn94VVRxPnnmH6cI28UwAL2uzuphNIuiuLOYKoIqSDcxDuXGxrpglXy6p4eE4ssQ2NuLLNloqlZdmDAKjQwfrhWuGnKSTJxjX+XaPx38OY4utrDgK5xxnyZzFpa1NWZUiJXXhugEYxW3qknlGtjStTpwurWgQBVKONx1aOT7pDxBZ3Rc8ZCK+dhJHO3R6/lFxKCDDqHT+Te31rEar8GkrAZA23lI5mnFVpGIg2vazJPwP0AfQ6fcYtikecac6c8EpIRDglYGGpAa1VqkrkrLdapLht0xHgFVqlTD1qfm8uWiTy94mO89g8L+eR7a1vdR22wgZw+w7FBcHOK+ksErZVeKn6CA8H/OyjIfFht+w+3lOFE2J9rqmpyweb0nGfuFdJUF3rfYZH5zc5Y3KSnddUAggkjNuzutXqeapPPwupTAKi7c5Ql+5ugo/wpGzYqXhKbZXqWIZR+YjpB9Ic2hb7qLqXVuyt9jbl/vWamtKwBQWnQYciGXo5qqZTmJJlNfzfhzxi02IMgwSzZMpjjIc3ZwOcp1C5CZX4Ofi3sPwEAAP//muWKWA==" } diff --git a/x-pack/filebeat/module/azure/module.yml b/x-pack/filebeat/module/azure/module.yml index d71d79b7561..8b137891791 100644 --- a/x-pack/filebeat/module/azure/module.yml +++ b/x-pack/filebeat/module/azure/module.yml @@ -1,3 +1 @@ -dashboards: -- id: Filebeat-azure-auditlogs-Dashboard - file: Filebeat-azure-audit.json + diff --git a/x-pack/filebeat/module/azure/signinlogs/_meta/fields.yml b/x-pack/filebeat/module/azure/signinlogs/_meta/fields.yml index dfabc236891..2bbadc43e8e 100644 --- a/x-pack/filebeat/module/azure/signinlogs/_meta/fields.yml +++ b/x-pack/filebeat/module/azure/signinlogs/_meta/fields.yml @@ -1,7 +1,7 @@ - name: signinlogs type: group description: > - Fields for azure audit logs. + Fields for Azure sign-in logs. fields: - name: resourceID type: keyword diff --git a/x-pack/filebeat/module/azure/signinlogs/config/signinlogs.yml b/x-pack/filebeat/module/azure/signinlogs/config/signinlogs.yml index 0b8c0b3237b..c22e189eb94 100644 --- a/x-pack/filebeat/module/azure/signinlogs/config/signinlogs.yml +++ b/x-pack/filebeat/module/azure/signinlogs/config/signinlogs.yml @@ -7,8 +7,3 @@ expand_event_list_from_field: "records" username: "$ConnectionString" password: {{ .connection_string }} ssl.enabled: true - -processors: -- decode_json_fields: - fields: ["message"] - target: "azure.signinlogs" diff --git a/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.json b/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.json index e55c86a64a8..d3583ee85fa 100644 --- a/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.json +++ b/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.json @@ -1,6 +1,12 @@ { "description": "Pipeline for parsing azure signin logs.", "processors": [ + { + "json" : { + "field" : "message", + "target_field" : "azure.signinlogs" + } + }, { "date": { "field": "azure.signinlogs.time", diff --git a/x-pack/filebeat/module/cisco/shared/gen-ecs-mapping-docs.go b/x-pack/filebeat/module/cisco/shared/gen-ecs-mapping-docs.go index a6d54e5a8bf..1329731d665 100644 --- a/x-pack/filebeat/module/cisco/shared/gen-ecs-mapping-docs.go +++ b/x-pack/filebeat/module/cisco/shared/gen-ecs-mapping-docs.go @@ -14,8 +14,6 @@ import ( "os" "sort" - "github.com/elastic/beats/libbeat/common" - "github.com/pkg/errors" ) @@ -41,7 +39,7 @@ var outputTables = []struct { type idMappings map[string]fieldMappings -type fieldMappings map[string]common.StringSet +type fieldMappings map[string]StringSet func main() { if err := generate(); err != nil { diff --git a/x-pack/filebeat/modules.d/azure.yml.disabled b/x-pack/filebeat/modules.d/azure.yml.disabled index e94149040a9..61f827236cc 100644 --- a/x-pack/filebeat/modules.d/azure.yml.disabled +++ b/x-pack/filebeat/modules.d/azure.yml.disabled @@ -6,9 +6,13 @@ activitylogs: enabled: true var: + # Azure event hub namespace FQDN for example "eventhubs.servicebus.windows.net:9093" namespace: "" + # Eventhub name containing the activity logs, overwrite he default value if the logs are exported in a different eventhub eventhub: ["insights-operational-logs"] + # Consumer group name that has access to the event hub, we advise creating a dedicated consumer group for the azure module consumer_group: "$Default" + # the connection string required to communicate with Event Hubs, steps to generate one here https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string connection_string: "" # auditlogs: From 3d7452a1f0ece7f000f2ec93c6d964060392f461 Mon Sep 17 00:00:00 2001 From: Mariana Date: Wed, 9 Oct 2019 17:11:42 +0200 Subject: [PATCH 18/21] Revert local changes --- filebeat/filebeat.yml | 4 +--- filebeat/input/kafka/input.go | 3 +-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/filebeat/filebeat.yml b/filebeat/filebeat.yml index 067af19c22d..d02b4d161fb 100644 --- a/filebeat/filebeat.yml +++ b/filebeat/filebeat.yml @@ -11,9 +11,7 @@ # configuration file. #=========================== Filebeat inputs ============================= -logging.level: debug -filebeat.overwrite_pipelines: true -setup.ilm.enabled: false + filebeat.inputs: # Each - is an input. Most options can be set at the input level, so diff --git a/filebeat/input/kafka/input.go b/filebeat/input/kafka/input.go index 809190830cd..6feb5cfd742 100644 --- a/filebeat/input/kafka/input.go +++ b/filebeat/input/kafka/input.go @@ -348,9 +348,8 @@ func (h *groupHandler) parseMultipleMessages(bMessage []byte) []string { for _, ms := range obj[h.expandEventListFromField] { js, err := json.Marshal(ms) if err == nil { - messages = append(messages, string(js)) - }else { h.log.Errorw(fmt.Sprintf("Kafka serializing message %s", ms), "error", err) + messages = append(messages, string(js)) } } } From a5b35345a3c1fe125a69c7ef988cb9d0c2b36c58 Mon Sep 17 00:00:00 2001 From: Mariana Date: Wed, 9 Oct 2019 17:18:00 +0200 Subject: [PATCH 19/21] Revert change --- x-pack/filebeat/module/cisco/shared/gen-ecs-mapping-docs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/filebeat/module/cisco/shared/gen-ecs-mapping-docs.go b/x-pack/filebeat/module/cisco/shared/gen-ecs-mapping-docs.go index 1329731d665..62fc5f41914 100644 --- a/x-pack/filebeat/module/cisco/shared/gen-ecs-mapping-docs.go +++ b/x-pack/filebeat/module/cisco/shared/gen-ecs-mapping-docs.go @@ -39,7 +39,7 @@ var outputTables = []struct { type idMappings map[string]fieldMappings -type fieldMappings map[string]StringSet +type fieldMappings map[string]stringSet func main() { if err := generate(); err != nil { From 8fb458102118853f4b869401620b5fe0001cc89c Mon Sep 17 00:00:00 2001 From: Mariana Date: Wed, 9 Oct 2019 23:30:33 +0200 Subject: [PATCH 20/21] Progress on defining azure module fields --- filebeat/docs/fields.asciidoc | 199 +++++------- x-pack/filebeat/module/azure/_meta/fields.yml | 5 + .../azure/activitylogs/_meta/fields.yml | 42 ++- .../azure/activitylogs/ingest/pipeline.json | 87 +++++- .../test/activity_log_expected.json | 44 +-- x-pack/filebeat/module/azure/fields.go | 2 +- .../module/azure/signinlogs/_meta/fields.yml | 111 +++---- .../azure/signinlogs/ingest/pipeline.json | 290 +++++++++++++++++- .../signinlogs/test/signin_log_expected.json | 119 ++++--- 9 files changed, 589 insertions(+), 310 deletions(-) diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 8f6d259e2ce..118e406c3e2 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -1255,6 +1255,16 @@ Azure Module +*`azure.resource_id`*:: ++ +-- +Resource ID + + +type: keyword + +-- + [float] === activitylogs @@ -1269,16 +1279,6 @@ The canonical user ID of the owner of the source bucket. -*`azure.activitylogs.identity.claims.*`*:: -+ --- -Claims - - -type: object - --- - [float] === authorization @@ -1293,7 +1293,7 @@ Node allocatable pods -*`azure.activitylogs.identity.authorization.evidence.roleAssignmentScope`*:: +*`azure.activitylogs.identity.authorization.evidence.role_assignment_scope`*:: + -- Role assignment scope @@ -1303,7 +1303,7 @@ type: keyword -- -*`azure.activitylogs.identity.authorization.evidence.roleDefinitionId`*:: +*`azure.activitylogs.identity.authorization.evidence.role_definition_id`*:: + -- Role definition ID @@ -1323,7 +1323,7 @@ type: keyword -- -*`azure.activitylogs.identity.authorization.evidence.roleAssignmentId`*:: +*`azure.activitylogs.identity.authorization.evidence.role_assignment_id`*:: + -- Role assignment ID @@ -1333,7 +1333,7 @@ type: keyword -- -*`azure.activitylogs.identity.authorization.evidence.principalId`*:: +*`azure.activitylogs.identity.authorization.evidence.principal_id`*:: + -- Principal ID @@ -1343,7 +1343,7 @@ type: keyword -- -*`azure.activitylogs.identity.authorization.evidence.principalType`*:: +*`azure.activitylogs.identity.authorization.evidence.principal_type`*:: + -- Principal type @@ -1373,7 +1373,7 @@ type: keyword -- -*`azure.activitylogs.correlationId`*:: +*`azure.activitylogs.correlation_id`*:: + -- Correlation ID @@ -1383,43 +1383,50 @@ type: keyword -- -*`azure.activitylogs.resourceID`*:: +*`azure.activitylogs.operation_name`*:: + -- -Resource ID +Operation name type: keyword -- -*`azure.activitylogs.operationName`*:: +*`azure.activitylogs.result_signature`*:: + -- -Operation name +Result signature type: keyword -- -*`azure.activitylogs.resultSignature`*:: +[float] +=== properties + +Properties + + + +*`azure.activitylogs.properties.service_request_id`*:: + -- -Result signature +Service Request Id type: keyword -- -*`azure.activitylogs.properties.*`*:: +*`azure.activitylogs.properties.status_code`*:: + -- -Properties +Status code -type: object +type: keyword -- @@ -1799,17 +1806,7 @@ Fields for Azure sign-in logs. -*`azure.signinlogs.resourceID`*:: -+ --- -Resource ID - - -type: keyword - --- - -*`azure.signinlogs.operationName`*:: +*`azure.signinlogs.operation_name`*:: + -- The operation name @@ -1819,7 +1816,7 @@ type: keyword -- -*`azure.signinlogs.operationVersion`*:: +*`azure.signinlogs.operation_version`*:: + -- The operation version @@ -1829,17 +1826,7 @@ type: keyword -- -*`azure.signinlogs.category`*:: -+ --- -Log category - - -type: keyword - --- - -*`azure.signinlogs.tenantId`*:: +*`azure.signinlogs.tenant_id`*:: + -- Tenant ID @@ -1849,7 +1836,7 @@ type: keyword -- -*`azure.signinlogs.resultSignature`*:: +*`azure.signinlogs.result_signature`*:: + -- Result signature @@ -1859,22 +1846,12 @@ type: keyword -- -*`azure.signinlogs.correlationId`*:: +*`azure.signinlogs.correlation_id`*:: + -- Correlation ID -type: keyword - --- - -*`azure.signinlogs.resultType`*:: -+ --- -Result type - - type: keyword -- @@ -1906,7 +1883,7 @@ type: keyword -- -*`azure.signinlogs.properties.createdDateTime`*:: +*`azure.signinlogs.properties.created_at`*:: + -- Created date time @@ -1916,7 +1893,7 @@ type: keyword -- -*`azure.signinlogs.properties.userDisplayName`*:: +*`azure.signinlogs.properties.user_display_name`*:: + -- User display name @@ -1926,7 +1903,7 @@ type: keyword -- -*`azure.signinlogs.properties.correlationId`*:: +*`azure.signinlogs.properties.correlation_id`*:: + -- Correlation ID @@ -1936,7 +1913,7 @@ type: keyword -- -*`azure.signinlogs.properties.userPrincipalName`*:: +*`azure.signinlogs.properties.user_principal_name`*:: + -- User principal name @@ -1946,7 +1923,7 @@ type: keyword -- -*`azure.signinlogs.properties.userId`*:: +*`azure.signinlogs.properties.user_id`*:: + -- User ID @@ -1956,7 +1933,7 @@ type: keyword -- -*`azure.signinlogs.properties.appId`*:: +*`azure.signinlogs.properties.app_id`*:: + -- App ID @@ -1966,7 +1943,7 @@ type: keyword -- -*`azure.signinlogs.properties.appDisplayName`*:: +*`azure.signinlogs.properties.app_display_name`*:: + -- App display name @@ -1976,7 +1953,7 @@ type: keyword -- -*`azure.signinlogs.properties.ipAddress`*:: +*`azure.signinlogs.properties.ip_address`*:: + -- Ip address @@ -1986,7 +1963,7 @@ type: keyword -- -*`azure.signinlogs.properties.clientAppUsed`*:: +*`azure.signinlogs.properties.client_app_used`*:: + -- Client app used @@ -1996,7 +1973,7 @@ type: keyword -- -*`azure.signinlogs.properties.conditionalAccessStatus`*:: +*`azure.signinlogs.properties.conditional_access_status`*:: + -- Conditional access status @@ -2006,7 +1983,7 @@ type: keyword -- -*`azure.signinlogs.properties.originalRequestId`*:: +*`azure.signinlogs.properties.original_request_id`*:: + -- Original request ID @@ -2016,7 +1993,7 @@ type: keyword -- -*`azure.signinlogs.properties.isInteractive`*:: +*`azure.signinlogs.properties.is_interactive`*:: + -- Is interactive @@ -2026,7 +2003,7 @@ type: keyword -- -*`azure.signinlogs.properties.tokenIssuerName`*:: +*`azure.signinlogs.properties.token_issuer_name`*:: + -- Token issuer name @@ -2036,7 +2013,7 @@ type: keyword -- -*`azure.signinlogs.properties.tokenIssuerType`*:: +*`azure.signinlogs.properties.token_issuer_type`*:: + -- Token issuer type @@ -2046,17 +2023,17 @@ type: keyword -- -*`azure.signinlogs.properties.processingTimeInMilliseconds`*:: +*`azure.signinlogs.properties.processing_time_ms`*:: + -- Processing time in milliseconds -type: keyword +type: float -- -*`azure.signinlogs.properties.riskDetail`*:: +*`azure.signinlogs.properties.risk_detail`*:: + -- Risk detail @@ -2066,7 +2043,7 @@ type: keyword -- -*`azure.signinlogs.properties.riskLevelAggregated`*:: +*`azure.signinlogs.properties.risk_level_aggregated`*:: + -- Risk level aggregated @@ -2076,7 +2053,7 @@ type: keyword -- -*`azure.signinlogs.properties.riskLevelDuringSignIn`*:: +*`azure.signinlogs.properties.risk_level_during_signin`*:: + -- Risk level during signIn @@ -2086,7 +2063,7 @@ type: keyword -- -*`azure.signinlogs.properties.riskState`*:: +*`azure.signinlogs.properties.risk_state`*:: + -- Risk state @@ -2096,7 +2073,7 @@ type: keyword -- -*`azure.signinlogs.properties.resourceDisplayName`*:: +*`azure.signinlogs.properties.resource_display_name`*:: + -- Resource display name @@ -2113,7 +2090,7 @@ Status -*`azure.signinlogs.properties.status.errorCode`*:: +*`azure.signinlogs.properties.status.error_code`*:: + -- Error code @@ -2123,101 +2100,77 @@ type: keyword -- -[float] -=== deviceDetail - -Status - - - -*`azure.signinlogs.properties.deviceDetail.deviceId`*:: +*`azure.signinlogs.properties.status.additional_details`*:: + -- -Device ID +Additional details type: keyword -- -*`azure.signinlogs.properties.deviceDetail.operatingSystem`*:: -+ --- -Operating system +[float] +=== device_detail +Status -type: keyword --- -*`azure.signinlogs.properties.deviceDetail.browser`*:: +*`azure.signinlogs.properties.device_detail.device_id`*:: + -- -Browser +Device ID type: keyword -- -[float] -=== location - -Status - - - -*`azure.signinlogs.properties.location.city`*:: +*`azure.signinlogs.properties.device_detail.operating_system`*:: + -- -City +Operating system type: keyword -- -*`azure.signinlogs.properties.location.state`*:: +*`azure.signinlogs.properties.device_detail.browser`*:: + -- -State +Browser type: keyword -- -*`azure.signinlogs.properties.location.countryOrRegion`*:: +*`azure.signinlogs.properties.device_detail.display_name`*:: + -- -Country or region +Display name type: keyword -- -[float] -=== geoCoordinates - -GeoCoordinates - - - -*`azure.signinlogs.properties.location.geoCoordinates.latitude`*:: +*`azure.signinlogs.properties.device_detail.trust_type`*:: + -- -Latitude +Trust type type: keyword -- -*`azure.signinlogs.properties.location.geoCoordinates.longitude`*:: +*`azure.signinlogs.properties.service_principal_id`*:: + -- -Longitude +Status type: keyword diff --git a/x-pack/filebeat/module/azure/_meta/fields.yml b/x-pack/filebeat/module/azure/_meta/fields.yml index 9b2efffbd48..26050a7727f 100644 --- a/x-pack/filebeat/module/azure/_meta/fields.yml +++ b/x-pack/filebeat/module/azure/_meta/fields.yml @@ -1,5 +1,6 @@ - key: azure title: "Azure" + release: beta description: > Azure Module fields: @@ -7,3 +8,7 @@ type: group description: > fields: + - name: resource_id + type: keyword + description: > + Resource ID diff --git a/x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml b/x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml index d8db6b53e8c..7d4ca2d0f6f 100644 --- a/x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml +++ b/x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml @@ -1,5 +1,6 @@ - name: activitylogs type: group + release: beta description: > Fields for Azure activity logs. fields: @@ -8,12 +9,6 @@ description: > The canonical user ID of the owner of the source bucket. fields: - - name: claims.* - type: object - object_type: keyword - object_type_mapping_type: "*" - description: > - Claims - name: authorization type: group description: > @@ -24,11 +19,11 @@ description: > Node allocatable pods fields: - - name: roleAssignmentScope + - name: role_assignment_scope type: keyword description: > Role assignment scope - - name: roleDefinitionId + - name: role_definition_id type: keyword description: > Role definition ID @@ -36,15 +31,15 @@ type: keyword description: > Role - - name: roleAssignmentId + - name: role_assignment_id type: keyword description: > Role assignment ID - - name: principalId + - name: principal_id type: keyword description: > Principal ID - - name: principalType + - name: principal_type type: keyword description: > Principal type @@ -56,26 +51,29 @@ type: keyword description: > Action - - name: correlationId + - name: correlation_id type: keyword description: > Correlation ID - - name: resourceID - type: keyword - description: > - Resource ID - - name: operationName + - name: operation_name type: keyword description: > Operation name - - name: resultSignature + - name: result_signature type: keyword description: > Result signature - - name: properties.* - type: object - object_type: keyword - object_type_mapping_type: "*" + - name: properties + type: group description: > Properties + fields: + - name: service_request_id + type: keyword + description: > + Service Request Id + - name: status_code + type: keyword + description: > + Status code diff --git a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json index 541a46f6802..2bc11ee7d3d 100644 --- a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json +++ b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json @@ -7,6 +7,12 @@ "target_field" : "azure.activitylogs" } }, + { + "drop": { + "if" : "ctx.azure.activitylogs.identity == null", + "ignore_failure": false + } + }, { "date": { "field": "azure.activitylogs.time", @@ -17,10 +23,15 @@ ] } }, + { + "remove": { + "field": ["message", "azure.activitylogs.time"] + } + }, { "rename": { "field": "azure.activitylogs.resourceId", - "target_field": "azure.activitylogs.resourceID", + "target_field": "azure.resource_id", "ignore_missing": true } }, @@ -76,14 +87,80 @@ } }, { - "remove": { - "field": ["message", "azure.activitylogs.time"], + "rename": { + "field": "azure.activitylogs.operationName", + "target_field": "azure.activitylogs.operation_name", "ignore_missing": true } }, { - "drop": { - "if" : "ctx.azure.activitylogs.identity == null || ctx.azure.activitylogs.identity.claims == null" + "rename": { + "field": "azure.activitylogs.resultSignature", + "target_field": "azure.activitylogs.result_signature", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.activitylogs.identity.authorization.evidence.roleAssignmentScope", + "target_field": "azure.activitylogs.identity.authorization.evidence.role_assignment_scope", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.activitylogs.identity.authorization.evidence.roleDefinitionId", + "target_field": "azure.activitylogs.identity.authorization.evidence.role_definition_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.activitylogs.identity.authorization.evidence.roleAssignmentId", + "target_field": "azure.activitylogs.identity.authorization.evidence.role_assignment_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.activitylogs.identity.authorization.evidence.principalId", + "target_field": "azure.activitylogs.identity.authorization.evidence.principal_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.activitylogs.identity.authorization.evidence.principalType", + "target_field": "azure.activitylogs.identity.authorization.evidence.principal_type", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.activitylogs.correlationId", + "target_field": "azure.activitylogs.correlation_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.activitylogs.properties.serviceRequestId", + "target_field": "azure.activitylogs.properties.service_request_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.activitylogs.properties.statusMessage", + "target_field": "message", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.activitylogs.properties.statusCode", + "target_field": "azure.activitylogs.properties.status_code", + "ignore_missing": true } } ], diff --git a/x-pack/filebeat/module/azure/activitylogs/test/activity_log_expected.json b/x-pack/filebeat/module/azure/activitylogs/test/activity_log_expected.json index 2dcea76b480..b1e06f171dc 100644 --- a/x-pack/filebeat/module/azure/activitylogs/test/activity_log_expected.json +++ b/x-pack/filebeat/module/azure/activitylogs/test/activity_log_expected.json @@ -1,7 +1,7 @@ { "_index" : "filebeat-8.0.0-2019.10.09", "_type" : "_doc", - "_id" : "2a7e2503-", + "_id" : "2a7e2503-d7e2-405a", "_score" : null, "_source" : { "agent" : { @@ -15,18 +15,19 @@ "level" : "Error" }, "source" : { - "ip" : "51.221.31.31" + "ip" : "11.71.231.51" }, "fileset" : { "name" : "activitylogs" }, + "message" : """{"error":{"code":"ParentResourceNotFound","message":"Can not perform requested operation on nested resource. Parent resource 'azurelsevents' not found."}}""", "geo" : { "name" : "global" }, "input" : { "type" : "kafka" }, - "@timestamp" : "2019-10-09T08:24:34.373Z", + "@timestamp" : "2019-10-09T19:39:22.822Z", "ecs" : { "version" : "1.1.0" }, @@ -38,7 +39,7 @@ "ProfileName: �\fname=default" ], "partition" : 3, - "offset" : 46716, + "offset" : 47915, "topic" : "insights-operational-logs", "key" : "" }, @@ -64,19 +65,19 @@ "outcome" : "Failure" }, "azure" : { + "resource_id" : "/SUBSCRIPTIONS/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/RESOURCEGROUPS/SA-HEMANT/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/AZURELSEVENTS/AUTHORIZATIONRULES/ROOTMANAGESHAREDACCESSKEY", "activitylogs" : { - "resourceID" : "/SUBSCRIPTIONS/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/RESOURCEGROUPS/SA-HEMANT/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/AZURELSEVENTS/AUTHORIZATIONRULES/ROOTMANAGESHAREDACCESSKEY", - "operationName" : "MICROSOFT.EVENTHUB/NAMESPACES/AUTHORIZATIONRULES/LISTKEYS/ACTION", - "resultSignature" : "Failed.NotFound", + "result_signature" : "Failed.NotFound", + "operation_name" : "MICROSOFT.EVENTHUB/NAMESPACES/AUTHORIZATIONRULES/LISTKEYS/ACTION", "identity" : { "authorization" : { "evidence" : { - "roleAssignmentScope" : "/subscriptions/2a7e2503-d7e2-405a-a84c-c333b9f7cb73", - "roleDefinitionId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "role_definition_id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb7", "role" : "Azure EventGrid Service BuiltIn Role", - "roleAssignmentId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", - "principalId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", - "principalType" : "ServicePrincipal" + "role_assignment_scope" : "/subscriptions/2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "role_assignment_id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "principal_type" : "ServicePrincipal", + "principal_id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73" }, "scope" : "/subscriptions/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/resourceGroups/sa-hemant/providers/Microsoft.EventHub/namespaces/azurelsevents/authorizationRules/RootManageSharedAccessKey", "action" : "Microsoft.EventHub/namespaces/authorizationRules/listKeys/action" @@ -85,29 +86,28 @@ "http://schemas.xmlsoap.org/ws/2005/05/identity/claims/nameidentifier" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", "ver" : "1.0", "http://schemas.microsoft.com/identity/claims/identityprovider" : "https://sts.windows.net/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/", - "aio" : "2a7e2503-d7e2-405a-a84c-c3", + "aio" : "42VgYChmaFjYVOd58PiHOdfi963fDAA=", "iss" : "https://sts.windows.net/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/", - "uti" : "7m-OZFTlhUSg2aXLucMuAA", + "uti" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", "aud" : "https://management.core.windows.net/", - "nbf" : "1570609174", + "nbf" : "1570649662", "appidacr" : "2", "http://schemas.microsoft.com/identity/claims/tenantid" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", "appid" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", "http://schemas.microsoft.com/identity/claims/objectidentifier" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", - "exp" : "1570638274", - "iat" : "1570609174" + "exp" : "1570678762", + "iat" : "1570649662" } }, - "correlationId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "correlation_id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", "properties" : { - "serviceRequestId" : null, - "statusMessage" : """{"error":{"code":"ParentResourceNotFound","message":"Can not perform requested operation on nested resource. Parent resource 'azurelsevents' not found."}}""", - "statusCode" : "NotFound" + "status_code" : "NotFound", + "service_request_id" : null } } } }, "sort" : [ - 1570609474373 + 1570649962822 ] } diff --git a/x-pack/filebeat/module/azure/fields.go b/x-pack/filebeat/module/azure/fields.go index 0a74d94369c..c8531c1feb6 100644 --- a/x-pack/filebeat/module/azure/fields.go +++ b/x-pack/filebeat/module/azure/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAzure returns asset data. // This is the base64 encoded gzipped contents of module/azure. func AssetAzure() string { - return "eJzsW91u2zgTve9TDHJZoH2AXHyAa3+7MNA2RX56WzDiRJkNTWpJyoH79AtK1k8sUmJi0e4uqqumUuYckjPDM0PmAzzh7hLYz1LjOwBLVuAlXCzczxfvADiaTFNhSclL+N87AIDqHXxRvBTuVx4IBTeX1asPINkGO3PusbsCLyHXqiz2/+Ox+dLMC1OZpS3ZnVC5aV/6rAYt188flX14UHo/gMYwOMsfe98eMumzIY7Skt29eBliM8HIPbePCBmTSlLGBJQGNaxXoB7APiKoZ4m6+cGoUmcI92X2hPbjgSEf5T7tTDDamI/vBx801NX9X5hZz+v6xY/6qyfcPSvNxz/7sWFFQTLf/87F+wvP9xPT4p5lxTk4JFbaR6XpJ3M2guPyLUkk/lfFEZgQKmOW3QuEQvEhnfDc98ni1jlOht6PprhG8n0N5ynefe5aCVwYQ7ncoLQ3mSpCw+iGEnaUVw7IPddKILCWAZhRCn3aK3wgSQ5jPcYlFWfewsN6FUX49CRfufxnmcfe2kfMY6FJZlQwcWqu3xrgV7G83Z06njqe1ofdMAyF2RSjCCb+JNLf8UdS+lHIi6Htdo9UWqNg3mwxhjyBuuzMHvpFG2JYb+4DtzkC9npvM4SpCtQVqa9sc7gUR8BeNWYrmNBoS2FvKJfMdipxniGXwoLxWu7izo3cEg7V0IgSilBBr1FAEwP51nJsXw1lccnJzq+JndVXCeL/kO86La6m/bf95DtqM0xUsxHYesy3uYpZzJX2lyFvwv6scr/VBtKiZJ79/5jhVhZHkuIZ08T5NoNiGPx92DfWmG1gh+1P1ZD1gqTZlp33Bewf9gFWZArBdp7Qn4nLoukL8BppmAmGs3KNzKSSLLXtKknbR6zWcGKqxpx3JlIjjtxnIlSeI/+0u0G9JW/5O4/z5MjhfgcmADNI3QHRPQOZTgCNimtKtC4ja9EGELN4S8mjx9IGjWWbYUdjYhObiUjQeruhMZ2jbbSGr0tydAvptoJoNdJbu0d8NOVBZJUYWSGuxrIeTLnwzGQChXS7hOHaeUYSwQq9nYliwblGE+q0zcgFqIAxsIZSaVC3df5pHOfOoO7aGtP+s1GcHgi5p9rxcZyhO/pljxjWQM0T2xuV+PydifLEDZyv+AzbUdj4zJGIYpNF4mgqwc8wjVeCB/i1gS3JErNOwaTYHtbyQelNrRY05kxzknkl8vbASr9xx2BFKFRmC6ZFECI2ePZqLSZNQRoP2MvSmKwFv1JURbFkRXHqDviiKGJ634cLf2qaw2Wf0BluN00dT3dhjNiAit30Ic20xgoA+NeFUlDqJiIXEUNTmjMRM4/8jPlHG/mUS5Kzt62d2Q8kfzeufzeuh5C/G9czYMedYpbCeorl44c8qPOjrmO9CXHts5i2N18nxeOa82foJ2YaXWmUtp24rEGAM4tVSzFIx4mf5GcDlcSJOhf4dXrwMapwrqmZUH99Sqlm5a6+QxnEDpVFc/S+/eVPDzn94VVRxPnnmH6cI28UwAL2uzuphNIuiuLOYKoIqSDcxDuXGxrpglXy6p4eE4ssQ2NuLLNloqlZdmDAKjQwfrhWuGnKSTJxjX+XaPx38OY4utrDgK5xxnyZzFpa1NWZUiJXXhugEYxW3qknlGtjStTpwurWgQBVKONx1aOT7pDxBZ3Rc8ZCK+dhJHO3R6/lFxKCDDqHT+Te31rEar8GkrAZA23lI5mnFVpGIg2vazJPwP0AfQ6fcYtikecac6c8EpIRDglYGGpAa1VqkrkrLdapLht0xHgFVqlTD1qfm8uWiTy94mO89g8L+eR7a1vdR22wgZw+w7FBcHOK+ksErZVeKn6CA8H/OyjIfFht+w+3lOFE2J9rqmpyweb0nGfuFdJUF3rfYZH5zc5Y3KSnddUAggkjNuzutXqeapPPwupTAKi7c5Ql+5ugo/wpGzYqXhKbZXqWIZR+YjpB9Ic2hb7qLqXVuyt9jbl/vWamtKwBQWnQYciGXo5qqZTmJJlNfzfhzxi02IMgwSzZMpjjIc3ZwOcp1C5CZX4Ofi3sPwEAAP//muWKWA==" + return "eJzUW81u2zgQvucpBr23D5DDAm6zCxjYNkWa9iow4lghQpNakkrgPv2C+qFli5TomLR3dYpjYb5Po/n5OKQ/wgvuboH8bhTeABhmON7Ch5X9/OEGQCFHovEWntCQGwCKulSsNkyKW/jjBgCgvRe+Stpwa2LDkFN92371EQTZ4t68vcyuxluolGzq/j8em4dmxqYUatmoEgtG3XeD0RfcvUk1/r/XdHc99IZgfTcBIaVhr8zsuKz0BGVM3V7HLorA/qt9ONhI1XtvAASL+Gl077EbxiwZRWGY2R18GWK5wMhej88IJRFSsJJwaDQqWN+B3IB5RpBvAtXwoffcU1O+oPl0ZMhH+cC5jXmWiv0mlsfkrjn+Ec9gr2+SIhDOZUkMeeIItaTac2eI6Jgsvlovl+i9aYlrJN9TOC/xHnNXkmNBtGaV2KIwhS5lHXoQmEmidz6SvR4kR9hTgHkKB8QpbphgFuQw1S/Feo8/rhBzlC9P8uQQuIonR+8/wpO1YqJkNeEXJ/t9QD6NpmVyLaJe7IFiKNmWGEUw+eE1PW6fM6X9LOTV1PYAW0qlkBN/yZiDXoD9srd7HBkDtKxRdcD2czro+8EuTOyOBFHDTWFTjJi9zEoA/tBaBr/lfR7YZzcMj7vVu2XI95DFJWGhUb2yEguF/zSoA8UuRfB3OPDQ4cB6asdRMsQ0uiglzZWILQAcAEzlbEOZidKyJ2lWa/UkwTpo90l5PS9IJzIefJn5LWliWq0sl5PT3fILlZ7WxGQEXj3mXVkkBiup/MuEd2H/LSu/1QHSoCDCTPLinMdtLYZeclcDf1ypBI76TspHjmg76YuvDSuX2GH7S6W4eyF5Sp6NvoD94/X7HdM1JztP6ifishrW7bRDmlaCqVcekOhc6qiz3RZpu07ny66aC95EpGYCecyEy6pC+nnXt9dswVMhhafdoBaCdFxxffTr+wRk9upuVsfn0jEz78IlEDH4yLJnj2Fb1IZsp0OUhSaWiEjQumtoRFVoBq3hG8ycPbV6bCGcRnrvwIrOljyIXJBGLkbv5qoeLIVwYjKBRbt7heFlekISgWIx8kS9olShDg33EnIBVsMc2ECp0ajcSOEygfNTo9qPUJbjZysp2zCkwbXhIccEA9mvPWJYAw1X7DhW4NsvwpsLz4q+4Ru8zsLGV45MFIcqEkdTcnoFN95zGuDnElsww4ixCiZHe1iLjVTbTi0orIiiTFStyOuBpXpnxyB1KFWSJdMqCBGbPL1aiylTkCcChqlPRNWC/1JWRbEkde1V/xn5reo6Zs5+/OIvTXP62hd0hu2mufPpZxgjNqFimz7kcWusAID/XSpdetcqIoeWNGcmZh75GfOHy3xWCSaSj62t2Y9MnDS4zrW9c+oUufDNeS80Ru5mukk31mKGulfb2LribuLswZ53ga59FvNOkbv0PW+MfIXJV6nQiviCZJpef+nsAyUG27lXkInt0EXf83x1JxGhthNHja9nMyKVe+Jmxa1v9kcwMntnQacckMrlmJ/dcbwgOKnrbNgBqT6Gzh+nlkRUmLK6IEG5k6J41BCy7zKFMxSmsI5pNOZKlRbEeh+8IPu0FbQ9x0Z4QcoStS66Ywm5MtjBQQcHATgnbxSrmGWX++DGfQ8Eaji5EQ5qpgsmDKp2JyRTSK81zGE45SVfUBRM6wZVxgR7tDDQwcxn2AGhwFw9NaHZHbJaSRtoTFSFbanFNhzbGy69rT2Cz3eH0jZuYAK2jHOm0aZYOL4V0y8FRUMYz+OoB6ZfIABwQILjK/KCVJXCyqqQjHRaKJiB8hCjjbKvsFOO2bl1aK1OXU/RDujZApYpyFtCfvuTnzzkb7LuNFVUp11oI2dMu3+ELEed2VdKqtCZO0i7kfWnxYIglpNI1LXgLk8vsP23cpgQwnSTNWwPTi7UqGu9z57dJfaQ71qopRFvPzCxpWqnDW7z87ofEGEGcaD3pOTb0hA6CavPM0BHQ9tQzUrMKPpAglGNNnNH91OeCbBYC0f1+6PLCz9+SHZg2H11828AAAD//yZi9Vg=" } diff --git a/x-pack/filebeat/module/azure/signinlogs/_meta/fields.yml b/x-pack/filebeat/module/azure/signinlogs/_meta/fields.yml index 2bbadc43e8e..12b580e73cc 100644 --- a/x-pack/filebeat/module/azure/signinlogs/_meta/fields.yml +++ b/x-pack/filebeat/module/azure/signinlogs/_meta/fields.yml @@ -3,38 +3,26 @@ description: > Fields for Azure sign-in logs. fields: - - name: resourceID - type: keyword - description: > - Resource ID - - name: operationName + - name: operation_name type: keyword description: > The operation name - - name: operationVersion + - name: operation_version type: keyword description: > The operation version - - name: category - type: keyword - description: > - Log category - - name: tenantId + - name: tenant_id type: keyword description: > Tenant ID - - name: resultSignature + - name: result_signature type: keyword description: > Result signature - - name: correlationId + - name: correlation_id type: keyword description: > Correlation ID - - name: resultType - type: keyword - description: > - Result type - name: identity type: keyword description: > @@ -48,83 +36,83 @@ type: keyword description: > ID - - name: createdDateTime + - name: created_at type: keyword description: > Created date time - - name: userDisplayName + - name: user_display_name type: keyword description: > User display name - - name: correlationId + - name: correlation_id type: keyword description: > Correlation ID - - name: userPrincipalName + - name: user_principal_name type: keyword description: > User principal name - - name: userId + - name: user_id type: keyword description: > User ID - - name: appId + - name: app_id type: keyword description: > App ID - - name: appDisplayName + - name: app_display_name type: keyword description: > App display name - - name: ipAddress + - name: ip_address type: keyword description: > Ip address - - name: clientAppUsed + - name: client_app_used type: keyword description: > Client app used - - name: conditionalAccessStatus + - name: conditional_access_status type: keyword description: > Conditional access status - - name: originalRequestId + - name: original_request_id type: keyword description: > Original request ID - - name: isInteractive + - name: is_interactive type: keyword description: > Is interactive - - name: tokenIssuerName + - name: token_issuer_name type: keyword description: > Token issuer name - - name: tokenIssuerType + - name: token_issuer_type type: keyword description: > Token issuer type - - name: processingTimeInMilliseconds - type: keyword + - name: processing_time_ms + type: float description: > Processing time in milliseconds - - name: riskDetail + - name: risk_detail type: keyword description: > Risk detail - - name: riskLevelAggregated + - name: risk_level_aggregated type: keyword description: > Risk level aggregated - - name: riskLevelDuringSignIn + - name: risk_level_during_signin type: keyword description: > Risk level during signIn - - name: riskState + - name: risk_state type: keyword description: > Risk state - - name: resourceDisplayName + - name: resource_display_name type: keyword description: > Resource display name @@ -133,20 +121,24 @@ description: > Status fields: - - name: errorCode + - name: error_code type: keyword description: > Error code - - name: deviceDetail + - name: additional_details + type: keyword + description: > + Additional details + - name: device_detail type: group description: > Status fields: - - name: deviceId + - name: device_id type: keyword description: > Device ID - - name: operatingSystem + - name: operating_system type: keyword description: > Operating system @@ -154,33 +146,16 @@ type: keyword description: > Browser - - name: location - type: group - description: > - Status - fields: - - name: city - type: keyword - description: > - City - - name: state + - name: display_name type: keyword description: > - State - - name: countryOrRegion + Display name + - name: trust_type type: keyword description: > - Country or region - - name: geoCoordinates - type: group - description: > - GeoCoordinates - fields: - - name: latitude - type: keyword - description: > - Latitude - - name: longitude - type: keyword - description: > - Longitude + Trust type + - name: service_principal_id + type: keyword + description: > + Status + diff --git a/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.json b/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.json index d3583ee85fa..589b8ba1560 100644 --- a/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.json +++ b/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.json @@ -7,6 +7,11 @@ "target_field" : "azure.signinlogs" } }, + { + "drop": { + "if" : "ctx.azure.signinlogs.category != 'SignInLogs'" + } + }, { "date": { "field": "azure.signinlogs.time", @@ -20,7 +25,7 @@ { "rename": { "field": "azure.signinlogs.resourceId", - "target_field": "azure.signinlogs.resourceID", + "target_field": "azure.resource_id", "ignore_missing": true } }, @@ -76,14 +81,289 @@ } }, { - "remove": { - "field": ["message", "azure.signinlogs.time"], + "rename": { + "field": "azure.signinlogs.operationName", + "target_field": "azure.signinlogs.operation_name", "ignore_missing": true } }, { - "drop": { - "if" : "ctx.event.category != 'SignInLogs'" + "rename": { + "field": "azure.signinlogs.resultSignature", + "target_field": "azure.signinlogs.result_signature", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.operationVersion", + "target_field": "azure.signinlogs.operation_version", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.tenantId", + "target_field": "azure.signinlogs.tenant_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.correlationId", + "target_field": "azure.signinlogs.correlation_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.networkLocationDetails", + "target_field": "azure.signinlogs.properties.network_location_details", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.resourceId", + "target_field": "azure.signinlogs.properties.resource_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.appliedConditionalAccessPolicies", + "target_field": "azure.signinlogs.properties.applied_conditional_access_policies", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.deviceDetail", + "target_field": "azure.signinlogs.properties.device_detail", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.device_detail.deviceId", + "target_field": "azure.signinlogs.properties.device_detail.device_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.device_detail.operatingSystem", + "target_field": "azure.signinlogs.properties.device_detail.operating_system", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.device_detail.displayName", + "target_field": "azure.signinlogs.properties.device_detail.display_name", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.device_detail.trustType", + "target_field": "azure.signinlogs.properties.device_detail.trust_type", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.createdDateTime", + "target_field": "azure.signinlogs.properties.created_at", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.userDisplayName", + "target_field": "azure.signinlogs.properties.user_display_name", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.correlationId", + "target_field": "azure.signinlogs.properties.correlation_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.userPrincipalName", + "target_field": "azure.signinlogs.properties.user_principal_name", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.userId", + "target_field": "azure.signinlogs.properties.user_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.appId", + "target_field": "azure.signinlogs.properties.app_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.appDisplayName", + "target_field": "azure.signinlogs.properties.app_display_name", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.ipAddress", + "target_field": "azure.signinlogs.properties.ip_address", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.clientAppUsed", + "target_field": "azure.signinlogs.properties.client_app_used", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.conditionalAccessStatus", + "target_field": "azure.signinlogs.properties.conditional_access_status", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.originalRequestId", + "target_field": "azure.signinlogs.properties.original_request_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.isInteractive", + "target_field": "azure.signinlogs.properties.is_interactive", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.tokenIssuerName", + "target_field": "azure.signinlogs.properties.token_issuer_name", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.tokenIssuerType", + "target_field": "azure.signinlogs.properties.token_issuer_type", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.processingTimeInMilliseconds", + "target_field": "azure.signinlogs.properties.processing_time_ms", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.riskDetail", + "target_field": "azure.signinlogs.properties.risk_detail", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.riskLevelAggregated", + "target_field": "azure.signinlogs.properties.risk_level_aggregated", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.riskLevelDuringSignIn", + "target_field": "azure.signinlogs.properties.risk_level_during_signin", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.riskState", + "target_field": "azure.signinlogs.properties.risk_state", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.resourceDisplayName", + "target_field": "azure.signinlogs.properties.resource_display_name", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.status.errorCode", + "target_field": "azure.signinlogs.properties.status.error_code", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.status.additionalDetails", + "target_field": "azure.signinlogs.properties.status.additional_details", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.location.city", + "target_field": "geo.city_name", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.location.state", + "target_field": "geo.country_name", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.location.geoCoordinates.latitude", + "target_field": "geo.location.lat", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.location.geoCoordinates.longitude", + "target_field": "geo.location.lon", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.signinlogs.properties.servicePrincipalId", + "target_field": "azure.signinlogs.properties.service_principal_id", + "ignore_missing": true + } + }, + { + "remove": { + "field": ["message", "azure.signinlogs.time","azure.signinlogs.properties.location"], + "ignore_missing": true } } ], diff --git a/x-pack/filebeat/module/azure/signinlogs/test/signin_log_expected.json b/x-pack/filebeat/module/azure/signinlogs/test/signin_log_expected.json index 975095d8909..49658b5e70b 100644 --- a/x-pack/filebeat/module/azure/signinlogs/test/signin_log_expected.json +++ b/x-pack/filebeat/module/azure/signinlogs/test/signin_log_expected.json @@ -1,7 +1,7 @@ { - "_index" : "filebeat-8.0.0-2019.10.08", + "_index" : "filebeat-8.0.0-2019.10.09", "_type" : "_doc", - "_id" : "2a7e2503-d7e2-405a", + "_id" : "2a7e2503-d7e2-405a-a84c-c3", "_score" : null, "_source" : { "agent" : { @@ -15,18 +15,24 @@ "level" : 4 }, "source" : { - "ip" : "71.251.171.21" + "ip" : "91.171.71.101" }, "fileset" : { "name" : "signinlogs" }, "geo" : { - "country_iso_code" : "US" + "city_name" : "Paris", + "country_iso_code" : "FR", + "country_name" : "Paris", + "location" : { + "lon" : 2.341434343434343, + "lat" : 48.24353454545454 + } }, "input" : { "type" : "kafka" }, - "@timestamp" : "2019-10-08T21:32:19.991Z", + "@timestamp" : "2019-10-09T20:57:56.668Z", "ecs" : { "version" : "1.1.0" }, @@ -35,8 +41,8 @@ }, "kafka" : { "headers" : [ ], - "partition" : 0, - "offset" : 700, + "partition" : 2, + "offset" : 1326, "topic" : "insights-logs-signinlogs", "key" : "" }, @@ -44,7 +50,7 @@ "hostname" : "DESKTOP-RFOOE09", "os" : { "build" : "18362.388", - "kernel" : "10.0.183.388 (WinBuild.160101.0800)", + "kernel" : "10.0.18362.388 (WinBuild.160101.0800)", "name" : "Windows 10 Pro", "family" : "windows", "version" : "10.0", @@ -63,74 +69,59 @@ }, "azure" : { "signinlogs" : { - "resourceID" : "/tenants/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/providers/Microsoft.aadiam", - "operationName" : "Sign-in activity", - "resultSignature" : "None", - "operationVersion" : "1.0", + "tenant_id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "operation_version" : "1.0", + "result_signature" : "None", + "operation_name" : "Sign-in activity", "identity" : "Test Test", - "tenantId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", - "correlationId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "correlation_id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", "properties" : { - "networkLocationDetails" : [ ], - "resourceId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", - "appliedConditionalAccessPolicies" : [ ], - "deviceDetail" : { - "displayName" : "DESKTOP-MJBCS48", - "trustType" : "Azure AD joined", - "deviceId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", - "operatingSystem" : "Windows" - }, - "isInteractive" : false, - "createdDateTime" : "2019-10-08T21:32:19.9914118+00:00", - "clientAppUsed" : "Mobile Apps and Desktop clients", - "riskLevelAggregated" : "none", - "authenticationProcessingDetails" : [ ], - "processingTimeInMilliseconds" : 96, - "appId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", - "correlationId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", - "id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", - "riskState" : "none", - "authenticationDetails" : [ + "risk_level_aggregated" : "none", + "applied_conditional_access_policies" : [ ], + "created_at" : "2019-10-09T20:57:56.6687852+00:00", + "risk_level_during_signin" : "none", + "authenticationProcessingDetails" : [ { - "authenticationStepDateTime" : "2019-10-08T21:32:19.9914118+00:00", - "authenticationStepResultDetail" : "None", - "authenticationStepRequirement" : "Multi Factor Authentication", - "succeeded" : false + "value" : "True", + "key" : "Domain Hint Present" } ], - "userPrincipalName" : "test.test@elastic.co", - "servicePrincipalId" : "", - "tokenIssuerType" : "AzureAD", - "resourceDisplayName" : "Windows Azure Active Directory", - "originalRequestId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "token_issuer_type" : "AzureAD", + "original_request_id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "app_id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "authenticationDetails" : [ ], + "network_location_details" : [ ], "riskEventTypes" : [ ], - "appDisplayName" : "Windows Sign In", - "riskLevelDuringSignIn" : "none", - "ipAddress" : "71.251.171.21", + "is_interactive" : false, + "service_principal_id" : "", "authenticationRequirementPolicies" : [ ], - "userDisplayName" : "Test Test", - "userId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", - "tokenIssuerName" : "", - "riskDetail" : "none", - "location" : { - "countryOrRegion" : "US", - "city" : "San Leandro", - "geoCoordinates" : { - "latitude" : 37.3444379492, - "longitude" : -123.2354456456 - }, - "state" : "California" + "app_display_name" : "Microsoft Teams Web Client", + "ip_address" : "91.171.71.101", + "device_detail" : { + "device_id" : "", + "browser" : "Chrome 77.0.3865", + "operating_system" : "MacOs" }, + "risk_detail" : "none", + "token_issuer_name" : "", + "resource_display_name" : "Microsoft Teams Services", + "risk_state" : "none", + "user_principal_name" : "test@elastic.co", + "processing_time_ms" : 601, + "user_id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "resource_id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "correlation_id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "user_display_name" : "Test Test", "status" : { - "errorCode" : 0, - "additionalDetails" : "MFA claim has expired due to the policies configured on tenant" - }, - "mfaDetail" : { } + "error_code" : 0 + } } - } + }, + "resource_id" : "/tenants/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/providers/Microsoft.aadiam" } }, "sort" : [ - 1570570339991 + 1570654676668 ] } From 617b3e5c90ecab626f704ae0432d865d61b83772 Mon Sep 17 00:00:00 2001 From: Mariana Date: Thu, 10 Oct 2019 10:41:07 +0200 Subject: [PATCH 21/21] Work on auditlog event format --- filebeat/docs/fields.asciidoc | 78 +++++------- .../module/azure/auditlogs/_meta/fields.yml | 40 +++--- .../azure/auditlogs/ingest/pipeline.json | 118 +++++++++++++++++- .../auditlogs/test/audit_log_expected.json | 85 +++++++------ x-pack/filebeat/module/azure/fields.go | 2 +- 5 files changed, 210 insertions(+), 113 deletions(-) diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 118e406c3e2..203003bdb35 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -1437,17 +1437,7 @@ Fields for Azure audit logs. -*`azure.auditlogs.resourceID`*:: -+ --- -Resource ID - - -type: keyword - --- - -*`azure.auditlogs.operationName`*:: +*`azure.auditlogs.operation_name`*:: + -- The operation name @@ -1457,7 +1447,7 @@ type: keyword -- -*`azure.auditlogs.operationVersion`*:: +*`azure.auditlogs.operation_version`*:: + -- The operation version @@ -1467,17 +1457,7 @@ type: keyword -- -*`azure.auditlogs.category`*:: -+ --- -Log category - - -type: keyword - --- - -*`azure.auditlogs.tenantId`*:: +*`azure.auditlogs.tenant_id`*:: + -- Tenant ID @@ -1487,7 +1467,7 @@ type: keyword -- -*`azure.auditlogs.resultSignature`*:: +*`azure.auditlogs.result_signature`*:: + -- Result signature @@ -1497,7 +1477,7 @@ type: keyword -- -*`azure.auditlogs.correlationId`*:: +*`azure.auditlogs.correlation_id`*:: + -- Correlation ID @@ -1524,7 +1504,7 @@ type: keyword -- -*`azure.auditlogs.properties.activityDisplayName`*:: +*`azure.auditlogs.properties.activity_display_name`*:: + -- Activity display name @@ -1534,7 +1514,7 @@ type: keyword -- -*`azure.auditlogs.properties.resultReason`*:: +*`azure.auditlogs.properties.result_reason`*:: + -- Reason for the log result @@ -1544,7 +1524,7 @@ type: keyword -- -*`azure.auditlogs.properties.correlationId`*:: +*`azure.auditlogs.properties.correlation_id`*:: + -- Correlation ID @@ -1554,7 +1534,7 @@ type: keyword -- -*`azure.auditlogs.properties.loggedByService`*:: +*`azure.auditlogs.properties.logged_by_service`*:: + -- Logged by service @@ -1564,7 +1544,7 @@ type: keyword -- -*`azure.auditlogs.properties.operationType`*:: +*`azure.auditlogs.properties.operation_type`*:: + -- Operation type @@ -1605,13 +1585,13 @@ type: keyword -- [float] -=== targetResources +=== target_resources Target resources -*`azure.auditlogs.properties.targetResources.displayName`*:: +*`azure.auditlogs.properties.target_resources.display_name`*:: + -- Display name @@ -1621,7 +1601,7 @@ type: keyword -- -*`azure.auditlogs.properties.targetResources.id`*:: +*`azure.auditlogs.properties.target_resources.id`*:: + -- ID @@ -1631,7 +1611,7 @@ type: keyword -- -*`azure.auditlogs.properties.targetResources.type`*:: +*`azure.auditlogs.properties.target_resources.type`*:: + -- Type @@ -1641,7 +1621,7 @@ type: keyword -- -*`azure.auditlogs.properties.targetResources.ipAddress`*:: +*`azure.auditlogs.properties.target_resources.ip_address`*:: + -- ip Address @@ -1651,7 +1631,7 @@ type: keyword -- -*`azure.auditlogs.properties.targetResources.userPrincipalName`*:: +*`azure.auditlogs.properties.target_resources.user_principal_name`*:: + -- User principal name @@ -1662,13 +1642,13 @@ type: keyword -- [float] -=== modifiedProperties +=== modified_properties Modified properties -*`azure.auditlogs.properties.targetResources.modifiedProperties.newValue`*:: +*`azure.auditlogs.properties.target_resources.modified_properties.newValue`*:: + -- New value @@ -1678,7 +1658,7 @@ type: keyword -- -*`azure.auditlogs.properties.targetResources.modifiedProperties.displayName`*:: +*`azure.auditlogs.properties.target_resources.modified_properties.displayName`*:: + -- Display value @@ -1688,7 +1668,7 @@ type: keyword -- -*`azure.auditlogs.properties.targetResources.modifiedProperties.oldValue`*:: +*`azure.auditlogs.properties.target_resources.modified_properties.oldValue`*:: + -- Old value @@ -1699,7 +1679,7 @@ type: keyword -- [float] -=== initiatedBy +=== initiated_by Information regarding the initiator @@ -1712,7 +1692,7 @@ App -*`azure.auditlogs.properties.initiatedBy.app.servicePrincipalName`*:: +*`azure.auditlogs.properties.initiated_by.app.servicePrincipalName`*:: + -- Service principal name @@ -1722,7 +1702,7 @@ type: keyword -- -*`azure.auditlogs.properties.initiatedBy.app.displayName`*:: +*`azure.auditlogs.properties.initiated_by.app.displayName`*:: + -- Display name @@ -1732,7 +1712,7 @@ type: keyword -- -*`azure.auditlogs.properties.initiatedBy.app.appId`*:: +*`azure.auditlogs.properties.initiated_by.app.appId`*:: + -- App ID @@ -1742,7 +1722,7 @@ type: keyword -- -*`azure.auditlogs.properties.initiatedBy.app.servicePrincipalId`*:: +*`azure.auditlogs.properties.initiated_by.app.servicePrincipalId`*:: + -- Service principal ID @@ -1759,7 +1739,7 @@ User -*`azure.auditlogs.properties.initiatedBy.user.userPrincipalName`*:: +*`azure.auditlogs.properties.initiated_by.user.userPrincipalName`*:: + -- User principal name @@ -1769,7 +1749,7 @@ type: keyword -- -*`azure.auditlogs.properties.initiatedBy.user.displayName`*:: +*`azure.auditlogs.properties.initiated_by.user.displayName`*:: + -- Display name @@ -1779,7 +1759,7 @@ type: keyword -- -*`azure.auditlogs.properties.initiatedBy.user.id`*:: +*`azure.auditlogs.properties.initiated_by.user.id`*:: + -- ID @@ -1789,7 +1769,7 @@ type: keyword -- -*`azure.auditlogs.properties.initiatedBy.user.ipAddress`*:: +*`azure.auditlogs.properties.initiated_by.user.ipAddress`*:: + -- ip Address diff --git a/x-pack/filebeat/module/azure/auditlogs/_meta/fields.yml b/x-pack/filebeat/module/azure/auditlogs/_meta/fields.yml index f792cfcd9e5..2df841208e5 100644 --- a/x-pack/filebeat/module/azure/auditlogs/_meta/fields.yml +++ b/x-pack/filebeat/module/azure/auditlogs/_meta/fields.yml @@ -3,31 +3,23 @@ description: > Fields for Azure audit logs. fields: - - name: resourceID - type: keyword - description: > - Resource ID - - name: operationName + - name: operation_name type: keyword description: > The operation name - - name: operationVersion + - name: operation_version type: keyword description: > The operation version - - name: category - type: keyword - description: > - Log category - - name: tenantId + - name: tenant_id type: keyword description: > Tenant ID - - name: resultSignature + - name: result_signature type: keyword description: > Result signature - - name: correlationId + - name: correlation_id type: keyword description: > Correlation ID @@ -40,23 +32,23 @@ type: keyword description: > Log result - - name: activityDisplayName + - name: activity_display_name type: keyword description: > Activity display name - - name: resultReason + - name: result_reason type: keyword description: > Reason for the log result - - name: correlationId + - name: correlation_id type: keyword description: > Correlation ID - - name: loggedByService + - name: logged_by_service type: keyword description: > Logged by service - - name: operationType + - name: operation_type type: keyword description: > Operation type @@ -72,12 +64,12 @@ type: keyword description: > category - - name: targetResources + - name: target_resources type: group description: > Target resources fields: - - name: displayName + - name: display_name type: keyword description: > Display name @@ -89,15 +81,15 @@ type: keyword description: > Type - - name: ipAddress + - name: ip_address type: keyword description: > ip Address - - name: userPrincipalName + - name: user_principal_name type: keyword description: > User principal name - - name: modifiedProperties + - name: modified_properties type: group description: > Modified properties @@ -114,7 +106,7 @@ type: keyword description: > Old value - - name: initiatedBy + - name: initiated_by type: group description: > Information regarding the initiator diff --git a/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.json b/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.json index fad8ddb8e43..d8826e03011 100644 --- a/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.json +++ b/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.json @@ -7,6 +7,11 @@ "target_field" : "azure.auditlogs" } }, + { + "drop": { + "if" : "ctx.azure.auditlogs.category != 'AuditLogs'" + } + }, { "date": { "field": "azure.auditlogs.time", @@ -20,7 +25,7 @@ { "rename": { "field": "azure.auditlogs.resourceId", - "target_field": "azure.auditlogs.resourceID", + "target_field": "azure.resource_id", "ignore_missing": true } }, @@ -68,8 +73,115 @@ } }, { - "drop": { - "if" : "ctx.event.category != 'AuditLogs'" + "rename": { + "field": "azure.auditlogs.operationName", + "target_field": "azure.auditlogs.operation_name", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.auditlogs.resultSignature", + "target_field": "azure.auditlogs.result_signature", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.auditlogs.operationVersion", + "target_field": "azure.auditlogs.operation_version", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.auditlogs.tenantId", + "target_field": "azure.auditlogs.tenant_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.auditlogs.correlationId", + "target_field": "azure.auditlogs.correlation_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.auditlogs.properties.activityDisplayName", + "target_field": "azure.auditlogs.properties.activity_display_name", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.auditlogs.properties.resultReason", + "target_field": "azure.auditlogs.properties.result_reason", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.auditlogs.properties.correlationId", + "target_field": "azure.auditlogs.properties.correlation_id", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.auditlogs.properties.loggedByService", + "target_field": "azure.auditlogs.properties.logged_by_service", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.auditlogs.properties.operationType", + "target_field": "azure.auditlogs.properties.operation_type", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.auditlogs.properties.targetResources", + "target_field": "azure.auditlogs.properties.target_resources", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.auditlogs.properties.target_resources.displayName", + "target_field": "azure.auditlogs.properties.target_resources.display_name", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.auditlogs.properties.target_resources.ipAddress", + "target_field": "azure.auditlogs.properties.target_resources.ip_address", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.auditlogs.properties.target_resources.userPrincipalName", + "target_field": "azure.auditlogs.properties.target_resources.user_principal_name", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.auditlogs.properties.target_resources.modifiedProperties", + "target_field": "azure.auditlogs.properties.target_resources.modified_properties", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.auditlogs.properties.initiatedBy", + "target_field": "azure.auditlogs.properties.initiated_by", + "ignore_missing": true } } ], diff --git a/x-pack/filebeat/module/azure/auditlogs/test/audit_log_expected.json b/x-pack/filebeat/module/azure/auditlogs/test/audit_log_expected.json index ec57eda58c4..54c46b04d1a 100644 --- a/x-pack/filebeat/module/azure/auditlogs/test/audit_log_expected.json +++ b/x-pack/filebeat/module/azure/auditlogs/test/audit_log_expected.json @@ -1,5 +1,5 @@ { - "_index" : "filebeat-8.0.0-2019.10.09", + "_index" : "filebeat-8.0.0-2019.10.10", "_type" : "_doc", "_id" : "2a7e2503-d7e2-405", "_score" : null, @@ -20,7 +20,7 @@ "input" : { "type" : "kafka" }, - "@timestamp" : "2019-10-08T22:18:17.146Z", + "@timestamp" : "2019-10-10T07:57:40.301Z", "ecs" : { "version" : "1.1.0" }, @@ -29,8 +29,8 @@ }, "kafka" : { "headers" : [ ], - "partition" : 3, - "offset" : 28, + "partition" : 2, + "offset" : 48, "topic" : "insights-logs-auditlogs", "key" : "" }, @@ -57,55 +57,68 @@ }, "azure" : { "auditlogs" : { - "resourceID" : "/tenants/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/providers/Microsoft.aadiam", - "operationVersion" : "1.0", - "tenantId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", - "operationName" : "Disable account", - "correlationId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "operation_name" : "Add member to group", + "tenant_id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "operation_version" : "1.0", + "identity" : "Microsoft Office 365 Portal", + "correlation_id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "result_signature" : "None", "properties" : { - "activityDisplayName" : "Disable account", - "resultReason" : "", - "operationType" : "Update", - "correlationId" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", - "loggedByService" : "Core Directory", - "id" : "Directory_U6", - "activityDateTime" : "2019-10-08T22:18:17.1467655+00:00", - "additionalDetails" : [ ], - "category" : "UserManagement", - "targetResources" : [ + "logged_by_service" : "Core Directory", + "initiated_by" : { + "user" : { + "displayName" : "Microsoft Office 365 Portal", + "ipAddress" : null, + "id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "userPrincipalName" : "test.test@elastic.co" + } + }, + "activity_display_name" : "Add member to group", + "operation_type" : "Assign", + "correlation_id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", + "activityDateTime" : "2019-10-10T07:57:40.3016161+00:00", + "id" : "Directory_TOE2090", + "target_resources" : [ { "displayName" : null, "modifiedProperties" : [ { - "newValue" : "[false]", - "displayName" : "AccountEnabled", - "oldValue" : "[true]" + "newValue" : """"2a7e2503-d7e2-405a-a84c-c333b9f7cb73"""", + "displayName" : "Group.ObjectID", + "oldValue" : null + }, + { + "newValue" : """"cloud-developers"""", + "displayName" : "Group.DisplayName", + "oldValue" : null }, { - "newValue" : """"AccountEnabled"""", - "displayName" : "Included Updated Properties", + "newValue" : null, + "displayName" : "Group.WellKnownObjectName", "oldValue" : null } ], "id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", "type" : "User", "userPrincipalName" : "test.test@elastic.co" - } - ], - "initiatedBy" : { - "user" : { + }, + { + "groupType" : "azureAD", "displayName" : null, - "ipAddress" : null, + "modifiedProperties" : [ ], "id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", - "userPrincipalName" : "admintestsds@elastic365.onmicrosoft.com" + "type" : "Group" } - } - }, - "resultSignature" : "None" - } + ], + "additionalDetails" : [ ], + "category" : "GroupManagement", + "result_reason" : "" + } + }, + "resource_id" : "/tenants/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/providers/Microsoft.aadiam" } }, "sort" : [ - 1570573097146 + 1570694260301 ] -}, +} diff --git a/x-pack/filebeat/module/azure/fields.go b/x-pack/filebeat/module/azure/fields.go index c8531c1feb6..808e17f6395 100644 --- a/x-pack/filebeat/module/azure/fields.go +++ b/x-pack/filebeat/module/azure/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAzure returns asset data. // This is the base64 encoded gzipped contents of module/azure. func AssetAzure() string { - return "eJzUW81u2zgQvucpBr23D5DDAm6zCxjYNkWa9iow4lghQpNakkrgPv2C+qFli5TomLR3dYpjYb5Po/n5OKQ/wgvuboH8bhTeABhmON7Ch5X9/OEGQCFHovEWntCQGwCKulSsNkyKW/jjBgCgvRe+Stpwa2LDkFN92371EQTZ4t68vcyuxluolGzq/j8em4dmxqYUatmoEgtG3XeD0RfcvUk1/r/XdHc99IZgfTcBIaVhr8zsuKz0BGVM3V7HLorA/qt9ONhI1XtvAASL+Gl077EbxiwZRWGY2R18GWK5wMhej88IJRFSsJJwaDQqWN+B3IB5RpBvAtXwoffcU1O+oPl0ZMhH+cC5jXmWiv0mlsfkrjn+Ec9gr2+SIhDOZUkMeeIItaTac2eI6Jgsvlovl+i9aYlrJN9TOC/xHnNXkmNBtGaV2KIwhS5lHXoQmEmidz6SvR4kR9hTgHkKB8QpbphgFuQw1S/Feo8/rhBzlC9P8uQQuIonR+8/wpO1YqJkNeEXJ/t9QD6NpmVyLaJe7IFiKNmWGEUw+eE1PW6fM6X9LOTV1PYAW0qlkBN/yZiDXoD9srd7HBkDtKxRdcD2czro+8EuTOyOBFHDTWFTjJi9zEoA/tBaBr/lfR7YZzcMj7vVu2XI95DFJWGhUb2yEguF/zSoA8UuRfB3OPDQ4cB6asdRMsQ0uiglzZWILQAcAEzlbEOZidKyJ2lWa/UkwTpo90l5PS9IJzIefJn5LWliWq0sl5PT3fILlZ7WxGQEXj3mXVkkBiup/MuEd2H/LSu/1QHSoCDCTPLinMdtLYZeclcDf1ypBI76TspHjmg76YuvDSuX2GH7S6W4eyF5Sp6NvoD94/X7HdM1JztP6ifishrW7bRDmlaCqVcekOhc6qiz3RZpu07ny66aC95EpGYCecyEy6pC+nnXt9dswVMhhafdoBaCdFxxffTr+wRk9upuVsfn0jEz78IlEDH4yLJnj2Fb1IZsp0OUhSaWiEjQumtoRFVoBq3hG8ycPbV6bCGcRnrvwIrOljyIXJBGLkbv5qoeLIVwYjKBRbt7heFlekISgWIx8kS9olShDg33EnIBVsMc2ECp0ajcSOEygfNTo9qPUJbjZysp2zCkwbXhIccEA9mvPWJYAw1X7DhW4NsvwpsLz4q+4Ru8zsLGV45MFIcqEkdTcnoFN95zGuDnElsww4ixCiZHe1iLjVTbTi0orIiiTFStyOuBpXpnxyB1KFWSJdMqCBGbPL1aiylTkCcChqlPRNWC/1JWRbEkde1V/xn5reo6Zs5+/OIvTXP62hd0hu2mufPpZxgjNqFimz7kcWusAID/XSpdetcqIoeWNGcmZh75GfOHy3xWCSaSj62t2Y9MnDS4zrW9c+oUufDNeS80Ru5mukk31mKGulfb2LribuLswZ53ga59FvNOkbv0PW+MfIXJV6nQiviCZJpef+nsAyUG27lXkInt0EXf83x1JxGhthNHja9nMyKVe+Jmxa1v9kcwMntnQacckMrlmJ/dcbwgOKnrbNgBqT6Gzh+nlkRUmLK6IEG5k6J41BCy7zKFMxSmsI5pNOZKlRbEeh+8IPu0FbQ9x0Z4QcoStS66Ywm5MtjBQQcHATgnbxSrmGWX++DGfQ8Eaji5EQ5qpgsmDKp2JyRTSK81zGE45SVfUBRM6wZVxgR7tDDQwcxn2AGhwFw9NaHZHbJaSRtoTFSFbanFNhzbGy69rT2Cz3eH0jZuYAK2jHOm0aZYOL4V0y8FRUMYz+OoB6ZfIABwQILjK/KCVJXCyqqQjHRaKJiB8hCjjbKvsFOO2bl1aK1OXU/RDujZApYpyFtCfvuTnzzkb7LuNFVUp11oI2dMu3+ELEed2VdKqtCZO0i7kfWnxYIglpNI1LXgLk8vsP23cpgQwnSTNWwPTi7UqGu9z57dJfaQ71qopRFvPzCxpWqnDW7z87ofEGEGcaD3pOTb0hA6CavPM0BHQ9tQzUrMKPpAglGNNnNH91OeCbBYC0f1+6PLCz9+SHZg2H11828AAAD//yZi9Vg=" + return "eJzsW91u2zgTvc9TDHrfPkAuPsBfswsY2DaFm+6tQItjlTBNaknKgfv0C1I/liyRYhLRzgLVVRwLc47IOcPRIf0R9ni6B/KrUngHYJjheA8fVvbzhzsAhRyJxnvYoiF3ABR1rlhpmBT38L87AAB3L3yRtOI2xI4hp/reffURBDngOby9zKnEeyiUrMrmPxMxh2H6oRRqWakcM0a779qgezw9S9X//2To+to0gWD9MAIhuWFHZk5cFnqE0qdur8shisD+0z0c7KRqRq8FBIv4qXfv5TD0WTKKwjBzGnzpYznDyF5PPxFyIqRgOeFQaVSwfgC5A/MTQT4LVO2HZuS2Vb5H8+ki0BTlweBW5qdU7BexPEZ3hfhHPIO9vkqKQDiXOTFkyxFKSfXEnT6ifbJ4tKOc4+RNc1wj+b6E8xzvPnclOWZEa1aIAwqT6VyWvgeBgIhe+Uj22kiOcKYAYQoD4hR3TDALMpT6tVif8fsVIkT5+iRfnAI3Gcne/EeMZKmYyFlJ+NXJfmuRX0bTMrkV0UnslqJPbHOMIph8nwzdXz4Dpf1NyKtx7BY2l0ohJ9MlIwQ9A/v5HPcyM1poWaKqge3n5aAf27gwittriCpuMisxYs5t1gLgGxcZpiOfdWCf3TC8XK1e3YZ880Wcayw0qiPLMVP4T4XaU+yWSP4aBzY1DqzHcTpKhphKZ7mkqYToAGAAMG5nK8pMVC/7op7VRn1Rw5pKJbZxlfNKOcMfUelxhVqMwVT4loRBQaYW4reAu5C+0nTj+nDDorx8abLT3KW9P/5coaqnJE1B+EsWvviXb7cZZbrk5DSlxoXYrNr32gZqLM7xuGQKiU7VPmxcbFfF7Issnx+tYAIvxCqQzH0qXBYF0mx7ypqlLlkKFUhhewIfzLiieprgBdicW6Bgs5tqagLT0erogRh8YsklZNgBtSGHsdPQpSoxWEh1aQgtRMQbvVvaiCrQyre2h6bsizd7O08OA0IYMbbOTOWDyPe2yHe2h1Dxg7kkXpiM5922m0T/2+yCJJ58KN1IlBmhVKH2mWALkgFWwioA1nKqNKrs/N5/ndz5oVGdzYb5FDpIynYMaebtToYkF/AuvzSQ/oaovWKdS4HPfxNeXdlW+YrPcAzCXpSPr/4MSESxrSRxNCWnNxjGR049/DpxC2YYMa6VSbFIrMVOqkPdNCgsiKJMFK7fa5CleuW6QUqfVhZT08oLEauepmvrXMLrp2nrkETULXhPsopiScpywvFJym9VljGe9OXEX5vmeNpnmg27oqbW0w8/RqygLM0bqim2BYD/nJSuvcMToSFWhlrBZMwmWtCYPzrls0IwsbjFa8N+ZOK3yTvB4LfJ+y5M3uAhmFeBrqcipvWUa/m+zVS+gQGWK3RdPEnkZX+u4wMlBp395WXiXs3T29luJY6yst+RaxxnWyw1OjN9yoBUqoH5UR9d84KTskyG7WnV+9BX2HYpy7g0DdpsSxSPEnzxO6VwhsJkdmAqjamk4kDs6MMkyFm2grozX4RnJM9R66zewk+l4A4OajjwwHXtjWIFs+xSH3J4bIBAtacc/EnNdMaEQeU2RBKl9FpDCKPrvOQeRca0rlAlFNiThYEaJqywAaF0u2QDQsGNslJJm2hMFJldUrODP7d3XE4u7RF8vnUobuEGJuDAOGcarcT8+a2Y3mcUDWE8zUBtmN6DB2BAguMReUaKQmFhu5CEdBwUBKAmiNFK2SmsO8fk3Go016eux2gDeraAJUpyR2g6/ujnAekX2e4HBFEr7cwy8ga3+7svctT5dqWk8p1Pg2W3sv6wWODF6lok2i3BtU6vsAW46jDBh9k5a+gOGc7UqFvNZ8PuGhvJDw5qzuJtDBNbqk7a4CE9r8cWEQKILb2tks9zJvQirP4fAHrnpxKMqrQJHXNf8mCAxZo51t4c8535ocBih2u7r+7+DQAA//8dk8dD" }