Fix azure fields names (#14098)

* Fix azure fields names * Add changelog entry * Add claims fields * Add "claims.name" field (dashboards) * Create claims_initiated_by object * REmove dummy pipeline file * Fix condition on category and initiated_by fields
elastic · Oct 18, 2019 · 6f9dcd9 · 6f9dcd9
1 parent 21f6136
commit 6f9dcd9
Show file tree

Hide file tree

Showing 9 changed files with 165 additions and 16 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -171,6 +171,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix cisco module's asa and ftd filesets parsing of domain names where an IP address is expected. {issue}14034[14034]
 - Fixed increased memory usage with large files when multiline pattern does not match. {issue}14068[14068]
 - panw module: Use geo.name instead of geo.country_iso_code for free-form location. {issue}13272[13272]
+- Fix azure fields names. {pull}14098[14098]
 - Fix calculation of `network.bytes` and `network.packets` for bi-directional netflow events. {pull}14111[14111]
 
 *Heartbeat*

diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -1559,7 +1559,17 @@ type: keyword
 
 --
 
-*`azure.resource.type`*::
+*`azure.resource.provider`*::
++
+--
+Resource type/namespace
+
+
+type: keyword
+
+--
+
+*`azure.resource.namespace`*::
 +
 --
 Resource type/namespace
@@ -1589,21 +1599,78 @@ Fields for Azure activity logs.
 [float]
 === identity
 
-The canonical user ID of the owner of the source bucket.
+Identity
+
+
+
+[float]
+=== claims_initiated_by_user
 
+Claims initiated by user
+
+
+
+*`azure.activitylogs.identity.claims_initiated_by_user.name`*::
++
+--
+Name
+
+
+type: keyword
+
+--
+
+*`azure.activitylogs.identity.claims_initiated_by_user.givenname`*::
++
+--
+Givenname
 
 
+type: keyword
+
+--
+
+*`azure.activitylogs.identity.claims_initiated_by_user.surname`*::
++
+--
+Surname
+
+
+type: keyword
+
+--
+
+*`azure.activitylogs.identity.claims_initiated_by_user.fullname`*::
++
+--
+Fullname
+
+
+type: keyword
+
+--
+
+*`azure.activitylogs.identity.claims_initiated_by_user.schema`*::
++
+--
+Schema
+
+
+type: keyword
+
+--
+
 [float]
 === authorization
 
-Node allocatable pods
+Authorization
 
 
 
 [float]
 === evidence
 
-Node allocatable pods
+Evidence
 
 
 

diff --git a/x-pack/filebeat/module/azure/_meta/fields.yml b/x-pack/filebeat/module/azure/_meta/fields.yml
@@ -33,7 +33,11 @@
               type: keyword
               description: >
                 Resource group
-            - name: type
+            - name: provider
+              type: keyword
+              description: >
+                Resource type/namespace
+            - name: namespace
               type: keyword
               description: >
                 Resource type/namespace

diff --git a/x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml b/x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml
@@ -7,17 +7,42 @@
     - name: identity
       type: group
       description: >
-        The canonical user ID of the owner of the source bucket.
+        Identity
       fields:
+        - name: claims_initiated_by_user
+          type: group
+          description: >
+            Claims initiated by user
+          fields:
+            - name: name
+              type: keyword
+              description: >
+                Name
+            - name: givenname
+              type: keyword
+              description: >
+                Givenname
+            - name: surname
+              type: keyword
+              description: >
+                Surname
+            - name: fullname
+              type: keyword
+              description: >
+                Fullname
+            - name: schema
+              type: keyword
+              description: >
+                Schema
         - name: authorization
           type: group
           description: >
-            Node allocatable pods
+            Authorization
           fields:
             - name: evidence
               type: group
               description: >
-                Node allocatable pods
+                Evidence
               fields:
                 - name: role_assignment_scope
                   type: keyword

diff --git a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json
@@ -20,15 +20,16 @@
           "date": {
               "field": "azure.activitylogs.time",
               "target_field": "@timestamp",
-              "ignore_failure": false,
+              "ignore_failure": true,
               "formats": [
                   "ISO8601"
               ]
           }
       },
       {
           "remove": {
-              "field": ["message", "azure.activitylogs.time"]
+              "field": ["message", "azure.activitylogs.time"],
+              "ignore_missing": true
           }
       },
       {
@@ -79,10 +80,17 @@
       {
           "script": {
               "lang": "painless",
-              "source": "if (ctx.azure.activitylogs.properties.eventCategory == null) { if (ctx.azure.activitylogs.properties.policies != null)  { ctx.event.category = 'Policy';}} else {ctx.event.category = ctx.azure.activitylogs.properties.eventCategory;} if (ctx.event.category == null) {ctx.event.category='Administrative'}",
+              "source": "if (ctx.azure.activitylogs.properties != null && ctx.azure.activitylogs.properties.eventCategory != null) {ctx.eventCategory = ctx.azure.activitylogs.properties.eventCategory} if (ctx.azure.activitylogs.properties != null && ctx.azure.activitylogs.properties.policies != null)  { ctx.eventCategory = 'Policy'}  if (ctx.eventCategory == null) {ctx.eventCategory='Administrative'}",
               "ignore_failure": true
           }
       },
+      {
+          "rename": {
+              "field": "eventCategory",
+              "target_field": "event.category",
+              "ignore_missing": true
+          }
+      },
       {
           "rename": {
               "field": "azure.activitylogs.resultType",
@@ -167,6 +175,50 @@
               "ignore_missing": true
           }
       },
+      {
+          "geoip" : {
+              "field" : "source.ip",
+              "target_field" : "geo",
+              "ignore_missing": true
+          }
+      },
+      {
+          "rename": {
+              "field": "azure.activitylogs.identity.claims.name",
+              "target_field": "azure.activitylogs.identity.claims_initiated_by_user.fullname",
+              "ignore_missing": true
+          }
+      },
+      {
+          "script": {
+              "lang": "painless",
+              "source": "if (ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/surname'] != null) {  ctx.azure.activitylogs.identity.claims_initiated_by_user.surname = ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/surname'];}",
+              "ignore_failure": true
+          }
+      },
+      {
+          "script": {
+              "lang": "painless",
+              "source": "if (ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name'] != null) {  ctx.azure.activitylogs.identity.claims_initiated_by_user.name = ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name'];}",
+              "ignore_failure": true
+          }
+      },
+      {
+          "script": {
+              "lang": "painless",
+              "source": "if (ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname'] != null) {  ctx.azure.activitylogs.identity.claims_initiated_by_user.givenname = ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname'];}",
+              "ignore_failure": true
+          }
+      },
+      {
+          "set": {
+              "if" : "ctx.azure.activitylogs.identity!= null && ctx.azure.activitylogs.identity.claims_initiated_by_user != null && ctx.azure.activitylogs.identity.claims_initiated_by_user.name != null",
+              "field": "azure.activitylogs.identity.claims_initiated_by_user.schema",
+              "value": "http://schemas.xmlsoap.org/ws/2005/05/identity/claims",
+              "ignore_failure": true
+          }
+      },
+
       {
           "pipeline": {
               "name": "{< IngestPipeline "azure-shared-pipeline" >}"

diff --git a/x-pack/filebeat/module/azure/activitylogs/test/activity_log_expected.json b/x-pack/filebeat/module/azure/activitylogs/test/activity_log_expected.json
@@ -70,7 +70,7 @@
         "azure" : {
             "subscription_id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73",
             "resource" : {
-                "resource_group" : "SA-HEMANT",
+                "group" : "SA-HEMANT",
                 "provider" : "MICROSOFT.EVENTHUB",
                 "namespace" : "AZURELSEVENTS",
                 "id" : "/SUBSCRIPTIONS/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/RESOURCEGROUPS/SA-HEMANT/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/AZURELSEVENTS/AUTHORIZATIONRULES/ROOTMANAGESHAREDACCESSKEY",

diff --git a/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.json b/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.json
@@ -16,7 +16,7 @@
           "date": {
               "field": "azure.auditlogs.time",
               "target_field": "@timestamp",
-              "ignore_failure": false,
+              "ignore_failure": true,
               "formats": [
                   "ISO8601"
               ]

diff --git a/x-pack/filebeat/module/azure/azure-shared-pipeline.json b/x-pack/filebeat/module/azure/azure-shared-pipeline.json
@@ -10,7 +10,7 @@
         {
             "grok": {
                 "field": "azure.resource_id",
-                "patterns": ["/SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.resource_group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/NAMESPACES/%{NAMESPACE:azure.resource.namespace}/AUTHORIZATIONRULES/%{RULE:azure.resource.authorization_rule}"],
+                "patterns": ["/SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/NAMESPACES/%{NAMESPACE:azure.resource.namespace}/AUTHORIZATIONRULES/%{RULE:azure.resource.authorization_rule}"],
                 "pattern_definitions" : {
                     "SUBID" : "(\\{){0,1}[0-9a-fA-F]{8}\\-[0-9a-fA-F]{4}\\-[0-9a-fA-F]{4}\\-[0-9a-fA-F]{4}\\-[0-9a-fA-F]{12}(\\}){0,1}",
                     "GROUPID" : ".+",
@@ -24,7 +24,7 @@
         {
             "grok": {
                 "field": "azure.resource_id",
-                "patterns": ["/SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.resource_group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name}"],
+                "patterns": ["/SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name}"],
                 "pattern_definitions" : {
                     "SUBID" : "(\\{){0,1}[0-9a-fA-F]{8}\\-[0-9a-fA-F]{4}\\-[0-9a-fA-F]{4}\\-[0-9a-fA-F]{4}\\-[0-9a-fA-F]{12}(\\}){0,1}",
                     "GROUPID" : ".+",

diff --git a/x-pack/filebeat/module/azure/fields.go b/x-pack/filebeat/module/azure/fields.go