From 6f9dcd9e4bed7fca137a7e124d83d76db0c8a47e Mon Sep 17 00:00:00 2001 From: Mariana Dima Date: Fri, 18 Oct 2019 15:56:13 +0200 Subject: [PATCH] Fix azure fields names (#14098) * Fix azure fields names * Add changelog entry * Add claims fields * Add "claims.name" field (dashboards) * Create claims_initiated_by object * REmove dummy pipeline file * Fix condition on category and initiated_by fields --- CHANGELOG.next.asciidoc | 1 + filebeat/docs/fields.asciidoc | 75 ++++++++++++++++++- x-pack/filebeat/module/azure/_meta/fields.yml | 6 +- .../azure/activitylogs/_meta/fields.yml | 31 +++++++- .../azure/activitylogs/ingest/pipeline.json | 58 +++++++++++++- .../test/activity_log_expected.json | 2 +- .../azure/auditlogs/ingest/pipeline.json | 2 +- .../module/azure/azure-shared-pipeline.json | 4 +- x-pack/filebeat/module/azure/fields.go | 2 +- 9 files changed, 165 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 7837ade32b5..ee61ca60909 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -171,6 +171,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix cisco module's asa and ftd filesets parsing of domain names where an IP address is expected. {issue}14034[14034] - Fixed increased memory usage with large files when multiline pattern does not match. {issue}14068[14068] - panw module: Use geo.name instead of geo.country_iso_code for free-form location. {issue}13272[13272] +- Fix azure fields names. {pull}14098[14098] - Fix calculation of `network.bytes` and `network.packets` for bi-directional netflow events. {pull}14111[14111] *Heartbeat* diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index d7064fdd089..dc185a2f922 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -1559,7 +1559,17 @@ type: keyword -- -*`azure.resource.type`*:: +*`azure.resource.provider`*:: ++ +-- +Resource type/namespace + + +type: keyword + +-- + +*`azure.resource.namespace`*:: + -- Resource type/namespace @@ -1589,21 +1599,78 @@ Fields for Azure activity logs. [float] === identity -The canonical user ID of the owner of the source bucket. +Identity + + + +[float] +=== claims_initiated_by_user +Claims initiated by user + + + +*`azure.activitylogs.identity.claims_initiated_by_user.name`*:: ++ +-- +Name + + +type: keyword + +-- + +*`azure.activitylogs.identity.claims_initiated_by_user.givenname`*:: ++ +-- +Givenname +type: keyword + +-- + +*`azure.activitylogs.identity.claims_initiated_by_user.surname`*:: ++ +-- +Surname + + +type: keyword + +-- + +*`azure.activitylogs.identity.claims_initiated_by_user.fullname`*:: ++ +-- +Fullname + + +type: keyword + +-- + +*`azure.activitylogs.identity.claims_initiated_by_user.schema`*:: ++ +-- +Schema + + +type: keyword + +-- + [float] === authorization -Node allocatable pods +Authorization [float] === evidence -Node allocatable pods +Evidence diff --git a/x-pack/filebeat/module/azure/_meta/fields.yml b/x-pack/filebeat/module/azure/_meta/fields.yml index 9a284d34f24..792d5ef4f32 100644 --- a/x-pack/filebeat/module/azure/_meta/fields.yml +++ b/x-pack/filebeat/module/azure/_meta/fields.yml @@ -33,7 +33,11 @@ type: keyword description: > Resource group - - name: type + - name: provider + type: keyword + description: > + Resource type/namespace + - name: namespace type: keyword description: > Resource type/namespace diff --git a/x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml b/x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml index de258e799c2..fb781c4a6b4 100644 --- a/x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml +++ b/x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml @@ -7,17 +7,42 @@ - name: identity type: group description: > - The canonical user ID of the owner of the source bucket. + Identity fields: + - name: claims_initiated_by_user + type: group + description: > + Claims initiated by user + fields: + - name: name + type: keyword + description: > + Name + - name: givenname + type: keyword + description: > + Givenname + - name: surname + type: keyword + description: > + Surname + - name: fullname + type: keyword + description: > + Fullname + - name: schema + type: keyword + description: > + Schema - name: authorization type: group description: > - Node allocatable pods + Authorization fields: - name: evidence type: group description: > - Node allocatable pods + Evidence fields: - name: role_assignment_scope type: keyword diff --git a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json index dcfcd37889c..a41f2632e02 100644 --- a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json +++ b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json @@ -20,7 +20,7 @@ "date": { "field": "azure.activitylogs.time", "target_field": "@timestamp", - "ignore_failure": false, + "ignore_failure": true, "formats": [ "ISO8601" ] @@ -28,7 +28,8 @@ }, { "remove": { - "field": ["message", "azure.activitylogs.time"] + "field": ["message", "azure.activitylogs.time"], + "ignore_missing": true } }, { @@ -79,10 +80,17 @@ { "script": { "lang": "painless", - "source": "if (ctx.azure.activitylogs.properties.eventCategory == null) { if (ctx.azure.activitylogs.properties.policies != null) { ctx.event.category = 'Policy';}} else {ctx.event.category = ctx.azure.activitylogs.properties.eventCategory;} if (ctx.event.category == null) {ctx.event.category='Administrative'}", + "source": "if (ctx.azure.activitylogs.properties != null && ctx.azure.activitylogs.properties.eventCategory != null) {ctx.eventCategory = ctx.azure.activitylogs.properties.eventCategory} if (ctx.azure.activitylogs.properties != null && ctx.azure.activitylogs.properties.policies != null) { ctx.eventCategory = 'Policy'} if (ctx.eventCategory == null) {ctx.eventCategory='Administrative'}", "ignore_failure": true } }, + { + "rename": { + "field": "eventCategory", + "target_field": "event.category", + "ignore_missing": true + } + }, { "rename": { "field": "azure.activitylogs.resultType", @@ -167,6 +175,50 @@ "ignore_missing": true } }, + { + "geoip" : { + "field" : "source.ip", + "target_field" : "geo", + "ignore_missing": true + } + }, + { + "rename": { + "field": "azure.activitylogs.identity.claims.name", + "target_field": "azure.activitylogs.identity.claims_initiated_by_user.fullname", + "ignore_missing": true + } + }, + { + "script": { + "lang": "painless", + "source": "if (ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/surname'] != null) { ctx.azure.activitylogs.identity.claims_initiated_by_user.surname = ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/surname'];}", + "ignore_failure": true + } + }, + { + "script": { + "lang": "painless", + "source": "if (ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name'] != null) { ctx.azure.activitylogs.identity.claims_initiated_by_user.name = ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name'];}", + "ignore_failure": true + } + }, + { + "script": { + "lang": "painless", + "source": "if (ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname'] != null) { ctx.azure.activitylogs.identity.claims_initiated_by_user.givenname = ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname'];}", + "ignore_failure": true + } + }, + { + "set": { + "if" : "ctx.azure.activitylogs.identity!= null && ctx.azure.activitylogs.identity.claims_initiated_by_user != null && ctx.azure.activitylogs.identity.claims_initiated_by_user.name != null", + "field": "azure.activitylogs.identity.claims_initiated_by_user.schema", + "value": "http://schemas.xmlsoap.org/ws/2005/05/identity/claims", + "ignore_failure": true + } + }, + { "pipeline": { "name": "{< IngestPipeline "azure-shared-pipeline" >}" diff --git a/x-pack/filebeat/module/azure/activitylogs/test/activity_log_expected.json b/x-pack/filebeat/module/azure/activitylogs/test/activity_log_expected.json index 049d3439dc7..8178fe674a9 100644 --- a/x-pack/filebeat/module/azure/activitylogs/test/activity_log_expected.json +++ b/x-pack/filebeat/module/azure/activitylogs/test/activity_log_expected.json @@ -70,7 +70,7 @@ "azure" : { "subscription_id" : "2a7e2503-d7e2-405a-a84c-c333b9f7cb73", "resource" : { - "resource_group" : "SA-HEMANT", + "group" : "SA-HEMANT", "provider" : "MICROSOFT.EVENTHUB", "namespace" : "AZURELSEVENTS", "id" : "/SUBSCRIPTIONS/2a7e2503-d7e2-405a-a84c-c333b9f7cb73/RESOURCEGROUPS/SA-HEMANT/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/AZURELSEVENTS/AUTHORIZATIONRULES/ROOTMANAGESHAREDACCESSKEY", diff --git a/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.json b/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.json index e6558d54b5a..8c43d66a70b 100644 --- a/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.json +++ b/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.json @@ -16,7 +16,7 @@ "date": { "field": "azure.auditlogs.time", "target_field": "@timestamp", - "ignore_failure": false, + "ignore_failure": true, "formats": [ "ISO8601" ] diff --git a/x-pack/filebeat/module/azure/azure-shared-pipeline.json b/x-pack/filebeat/module/azure/azure-shared-pipeline.json index 73b7d3e7969..9bfad9cf1bb 100644 --- a/x-pack/filebeat/module/azure/azure-shared-pipeline.json +++ b/x-pack/filebeat/module/azure/azure-shared-pipeline.json @@ -10,7 +10,7 @@ { "grok": { "field": "azure.resource_id", - "patterns": ["/SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.resource_group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/NAMESPACES/%{NAMESPACE:azure.resource.namespace}/AUTHORIZATIONRULES/%{RULE:azure.resource.authorization_rule}"], + "patterns": ["/SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/NAMESPACES/%{NAMESPACE:azure.resource.namespace}/AUTHORIZATIONRULES/%{RULE:azure.resource.authorization_rule}"], "pattern_definitions" : { "SUBID" : "(\\{){0,1}[0-9a-fA-F]{8}\\-[0-9a-fA-F]{4}\\-[0-9a-fA-F]{4}\\-[0-9a-fA-F]{4}\\-[0-9a-fA-F]{12}(\\}){0,1}", "GROUPID" : ".+", @@ -24,7 +24,7 @@ { "grok": { "field": "azure.resource_id", - "patterns": ["/SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.resource_group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name}"], + "patterns": ["/SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name}"], "pattern_definitions" : { "SUBID" : "(\\{){0,1}[0-9a-fA-F]{8}\\-[0-9a-fA-F]{4}\\-[0-9a-fA-F]{4}\\-[0-9a-fA-F]{4}\\-[0-9a-fA-F]{12}(\\}){0,1}", "GROUPID" : ".+", diff --git a/x-pack/filebeat/module/azure/fields.go b/x-pack/filebeat/module/azure/fields.go index 2120b31c9a1..a84a96427fc 100644 --- a/x-pack/filebeat/module/azure/fields.go +++ b/x-pack/filebeat/module/azure/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAzure returns asset data. // This is the base64 encoded gzipped contents of module/azure. func AssetAzure() string { - return "eJzsW09v27gTvedTDHpvf/ccfoC32QUMbJMiTfcq0NJYJUKTWpJK4H76BSmJliVSYmJRyS6qQ9HExntPw/nHIfMRHvF4DeRnLfEKQFPN8Bo+bMzPH64AJDIkCq9hh5pcARSockkrTQW/hv9fAQDY78IXUdTMQOwpskJd248+AicHPMGbRx8rvIZSirpqf+PBPIfpQ6l6576d0cJ93gE/4vFZyP7vvfDN00jvQ8L2ZkSZCymRkUUYP5+wfFQaOeH6YpYGxkcgUYla5jjC7y/IDPr9GGO4Wn3Ks5eZeqEZ2j51/9X6XMO3WIpujOtW7FhhAkKD8T/DoCqSo5fZ/Lsc820fzcVtrukT1UcmSjXrMcNUEUH8h3Ub2AvZhmJHCIbxU7SDIddUH73G8HnEjCkefiDkhAtOc8KgVihhewNiD/oHgnjmKLsf2sXa1fkj6k8DIJ/kM+PW+oeQ9KdNBqNvTemPeAfz3IoCgTAmcqLJjiFUolCeb4aE9sXik7FyPnS4OK2Rel+ieU53X7sUDDOiFC35AbnOVC5GMTt+GX8UveKVzHMvGMJJAkxLOBNe4J5yOi4+a6k+8Q+Tbkjy+iJf7AJvYsne+kdYspKU57QibHWxXzvml8n0VMLVhHq5XcsYCLY5RRFKvnmh++VzIrVfxLwZY3e0okLZdKvLtgh3He649ei1lzXTmfFzok89/wLk9xYZ/MgnZzTvrikOS8are4GvIcS56q5QPtEcM4l/16gCGWcJD2x44L7hge0Yx0nSRNcqy0WRKhosAZwRjHvKuqA6qqF8UeNoUF/UNaaKEtM9ivlIOdE/oVTjNLGYAh/81HbzUvLR1hP+y/nB2Nr5Xhh/Lls0dkkTlX+KMoQ/3OdlBVUVI0dfSCykZtPt8Foq3zZ25C8SiUpVSO8ttk0lZkvH5q0VnActqCowJxpKYaIssch2x6ytN8lcqMQCdkcI0YzTWqAdXEDNqQ+ZbPtSLc3EcnRxdEM0PtDkIaTpAZUmh/Ge27kq0VgKORyNLCQkiO7qC5ElmvBtBiW+jfzFU44HywFTHDEDjpnMB5E7mMjdy81U8oM5J15YTGCXNzHhTCDiIcTiLFFlpCgkqtA4aEExQCvYTJB1mmqFMjvtgNfxne8K5WnbPe9CB1HQPcUiC3Yn5yIXmOJ9aSnDDVH3xM7wOD7/RVi98oDhFp/haZJ2kD5uwx6QSGKXSeJkCla8gRnvWBHQ54KbU02Jtq1MiiKx5XshD03TILEksqC8tP1eyyzkK+sGqUKxslg0bYIUsdHTdm1uXra+m3Zjioi8Be8prKJUkqryjF2S6ttUVcx0drjwa8scL/tMs2Eqaup4+h7miA0oI/MNoym2BYB/XSitfdYREUO0mmoFkynztKAx/3GRT0tO+eJzVgP7kfJfk1aPgl+T1sg7Ga+i3foQ0w52mxi6bLL7BlOoXKJtpUmigfLnBh8KotHOoIJK7P44/UzZlsOoefI7Gt3GzQ6Wss5Ms3AmKpVhvjc3qYLkpKqScQf65T71CmcfVRXnppOzriWSRwUhfBcpjCLXmTFMrTBVqFgSY33wkpzClhf2ChJhGclzVCprDrNTRbCjg4YOAnSux5C0pEZd6uP+u5YIZHfeH3ZqqjLKNUp7KpHIpbcKpjhc+yMekWdUqRplwgB7MDTQ0ExH2JmgdEdVZ4ImT6sqKYyjUV5mpqRmh7Bv75nwlvYIPV8diy3cQDkcKGNUoQmxsH9Lqh6zAjWhLI2h7ql6hADBmQiGT8gyUpYSS9OFJJRjqWCCyiOsqKVZwqZzTK6tYbN96nbMdibPJLBETm4F+fGHf3awQpF1V+ijKu1MGblg5PwthBx13VpKIUM3tWDZ86TfDRcEuVyLVLgS3MTpCudwG8cJIU433kJ73W4mR73Verbq1jjNvbFUc3PWdmphUtVRaTyk13XXMcIEYydvJ8Xz3CR4EVW/TRC986sBWtZKT926XvJ03nDN3LJuL7zO3Ftf7Jqp++jqnwAAAP//Aacrew==" + return "eJzsm9Fv3KgTx9/zV4z63t/vPQ8n7TXtKdK1qdL0Xi2ynnVQvIYDvNH2rz+B19hrAyYNOLlT/dTG0Xy+hplhGMh7eMTjJZAfrcALAEVVjZfwbqP//+4CQGCNROIl3KMiFwAlyq2gXFHWXMJvFwAA5nfhMyvbWpvYUaxLeWlevYeG7HEwrx915HgJlWAtP/3EYfPczNiUbO/tbxe0tO97w494fGJi/HOn+e7ppI9NwvXVDLllQmBNkhA/DLZcKIUNadSLKZ0ZF0CgZK3Y4sz+eEIWrN/ObUxna4w8+5jQBy1gx+jxp41Z069IhZvb7YlcsAMtUWSAahv/1xTJydloD3T321Xx6chfxtZs6tgqeqDqWLNKLjrtNFtFgD8Zz4UdE6ds0ANBE/8X7ePYKKqOzsFwOeXCUFy77bkkjGVsa0L3sqANVZQoLIv7Y9HKmXOGpUXI088HwwLLgvsjeFg+2RB2pXOpbpd6hmCYuphLRkUP2Kyj5Y8galjvxDpyvgVAvZhdW9frqPkUItmx2T7gnqwwNG6OTVGtemCC/jCreo5Q2ywAYuIL9SrVzBaKOI2ROvXzMcwJSR3LFazGgkhJq2aPjSrklnGfTYic7Wd8hX5uWY0wSICwhDPhJe5MXpwWjGupHvjTQskneX2Rz3aBVxnJ0fxHjCQXtNlSTurVxX7tyc+TqZW8llAne0jt7mBbUhSh5JvT9LjeDGTxF5E3c9s9lnEU3Q4zbU1909udF1ijLWFbq0L7OVHDPj0B/NZYBrfl0QaKo1AUZari+avP4lL5LFEc6BYLgX+3KD0ZJ4UHdhy47ThwPbdjJSmiWllsWZkrGgwAzgDzTVhbUhW1A3vWTktbfdY2K1eU3D3gYNsfKQP+gELO00QyBS7zoRbRS+GzdhH8l/ODHmvre377S9miG5c8Ufknq3z2p42RoqSS1+ToColEajZ9S+SEcm/WJ/4ikMhcC+mtsW1SiXpAM40Lo+Xt4SZU5entTqXUrKq61sxpvcnmQlXXlfFh5mnNUw4mUDPUIcGyL9fUBKajj6MrovCOZg8hRfcoFdnPt9nWVYnCiolp7y+REK91u74QUaEO364NPM2KkKKhcWcYEGLE9DQWMh+k7QJdhZIfLDlxYjGeXZ6dRP++LqGIOx/FjgQvSFkKlK4ZTiwGKIdNANZraiWKYtgBr+M73yWKYdu97EJ7VtIdxbLwVifnIhM07j6fkP6CqH9ie3gNPv1F6nblBsMXfIJDEDtJH94zgWwS+0wSJ5PV5SsM401devTZ4B6dMuVYJK6bHRP7rmgQWBFR0qYy9d6JzH72rIlwX6wki6aNFxEbPaeqzfbL1nfTvk0RkbfgLYVVlErCuaPtklXfhvOY7ux04teWOZ/2hWLDc+oLKePpu58RG1Ba5itGU2wJAP+6UFr7rCMihigPlYLZlDlK0Jh/2MinVUOb5H1WbfY9bX51Wh0KfnVaIy8x/RTWeY0pb2O3i6GXdXZfoQu1FWhKaZKpofyhsw8lUWh6UF4lZn+cv6dslsOofvIbat3G9Q5Sjc5CsXAmKtfAGCGh7inn2dieenmMXuHsg/M4Nw32ulIkDw4++8P1T4qNKvTAtBJzhYqB6NEHJ2QI26Y0V5BIXZDtFqUsusPsXBFscdDhwIOzNYagFdXqch/335xAIPrzfr9TU1nQRqEwpxKZXPpaQohhyx/2iE1BpWxRZAywO42BDhOOsDNB+Y6qzgQFT6u4YNrRaFMVekkt9n7f3tXMubRH6PlqKWbhBtrAntY1lahDzO/fgsrHokRFaJ1noG6pfAQP4ExEjQesC1JVAitdhWSUY1AQQDmEla3QU9hVjtm1dTRTp17PaWfydALL5ORGkNv+9E+FVlhk7d+cRK20C8vIC1rO33yWo25YC8GE76YWpD1P+qhZ4GXZEqm0S3AXpyucw20sE3xM295Cc91uIUe91nye1K1xmntlUEt91lPXQqeqo1S4z6/rpidCgNjLuxfsaakTnETV7wHQG78aoEQrVejWdcrTec1auGV9uvC6cG892TVT++rinwAAAP//Rgb1OQ==" }