[Metricbeat] Add bucket name to aws.s3.bucket.name (#13581)

* Add bucket name to aws.s3.bucket.name * Remove old bucket name fields * Update s3 bucket name field in dashboard * Change visualization version back to 7.0.0
elastic · Sep 13, 2019 · 102302b · 102302b
1 parent 79d75a3
commit 102302b
Show file tree

Hide file tree

Showing 10 changed files with 67 additions and 74 deletions.
diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc
@@ -844,6 +844,16 @@ type: object
 
 --
 
+*`aws.s3.bucket.name`*::
++
+--
+Name of a S3 bucket.
+
+
+type: keyword
+
+--
+
 [float]
 === cloudwatch
 
@@ -1784,16 +1794,6 @@ type: long
 
 
 
-*`aws.s3_daily_storage.bucket.name`*::
-+
---
-Name of a S3 bucket.
-
-
-type: keyword
-
---
-
 *`aws.s3_daily_storage.bucket.size.bytes`*::
 +
 --
@@ -1823,16 +1823,6 @@ type: long
 
 
 
-*`aws.s3_request.bucket.name`*::
-+
---
-Name of a S3 bucket.
-
-
-type: keyword
-
---
-
 *`aws.s3_request.requests.total`*::
 +
 --

diff --git a/x-pack/metricbeat/module/aws/_meta/fields.yml b/x-pack/metricbeat/module/aws/_meta/fields.yml
@@ -14,3 +14,7 @@
           object_type_mapping_type: "*"
           description: >
             Tag key value pairs from aws resources.
+        - name: s3.bucket.name
+          type: keyword
+          description: >
+            Name of a S3 bucket.
diff --git a/x-pack/metricbeat/module/aws/_meta/kibana/7/dashboard/Metricbeat-aws-s3-overview.json b/x-pack/metricbeat/module/aws/_meta/kibana/7/dashboard/Metricbeat-aws-s3-overview.json
@@ -29,7 +29,7 @@
             },
             "panelIndex": "1",
             "panelRefName": "panel_0",
-            "version": "7.0.0-beta1"
+            "version": "7.4.0"
           },
           {
             "embeddableConfig": {},
@@ -42,7 +42,7 @@
             },
             "panelIndex": "2",
             "panelRefName": "panel_1",
-            "version": "7.0.0-beta1"
+            "version": "7.4.0"
           },
           {
             "embeddableConfig": {},
@@ -55,7 +55,7 @@
             },
             "panelIndex": "3",
             "panelRefName": "panel_2",
-            "version": "7.0.0-beta1"
+            "version": "7.4.0"
           },
           {
             "embeddableConfig": {},
@@ -68,7 +68,7 @@
             },
             "panelIndex": "4",
             "panelRefName": "panel_3",
-            "version": "7.0.0-beta1"
+            "version": "7.4.0"
           },
           {
             "embeddableConfig": {},
@@ -81,7 +81,7 @@
             },
             "panelIndex": "5",
             "panelRefName": "panel_4",
-            "version": "7.0.0-beta1"
+            "version": "7.4.0"
           },
           {
             "embeddableConfig": {},
@@ -94,7 +94,7 @@
             },
             "panelIndex": "6",
             "panelRefName": "panel_5",
-            "version": "7.0.0-beta1"
+            "version": "7.4.0"
           },
           {
             "embeddableConfig": {},
@@ -107,7 +107,7 @@
             },
             "panelIndex": "7",
             "panelRefName": "panel_6",
-            "version": "7.0.0-beta1"
+            "version": "7.4.0"
           }
         ],
         "timeRestore": false,
@@ -116,7 +116,7 @@
       },
       "id": "a096b830-4762-11e9-8062-c98a86cb6f94",
       "migrationVersion": {
-        "dashboard": "7.0.0"
+        "dashboard": "7.3.0"
       },
       "references": [
         {
@@ -156,8 +156,8 @@
         }
       ],
       "type": "dashboard",
-      "updated_at": "2019-03-15T20:56:09.373Z",
-      "version": "WzkyNCw3XQ=="
+      "updated_at": "2019-09-12T21:18:24.987Z",
+      "version": "Wzc3NjAsN10="
     },
     {
       "attributes": {
@@ -242,8 +242,8 @@
       },
       "references": [],
       "type": "visualization",
-      "updated_at": "2019-03-15T20:23:22.120Z",
-      "version": "WzkxMSw3XQ=="
+      "updated_at": "2019-09-12T21:13:42.937Z",
+      "version": "Wzc0MzQsN10="
     },
     {
       "attributes": {
@@ -320,8 +320,8 @@
       },
       "references": [],
       "type": "visualization",
-      "updated_at": "2019-03-15T20:30:52.573Z",
-      "version": "WzkxMyw3XQ=="
+      "updated_at": "2019-09-12T21:13:42.937Z",
+      "version": "Wzc0MzUsN10="
     },
     {
       "attributes": {
@@ -413,8 +413,8 @@
       },
       "references": [],
       "type": "visualization",
-      "updated_at": "2019-03-15T20:37:36.827Z",
-      "version": "WzkxNCw3XQ=="
+      "updated_at": "2019-09-12T21:13:42.937Z",
+      "version": "Wzc0MzYsN10="
     },
     {
       "attributes": {
@@ -488,8 +488,8 @@
       },
       "references": [],
       "type": "visualization",
-      "updated_at": "2019-03-15T20:40:02.231Z",
-      "version": "WzkxNSw3XQ=="
+      "updated_at": "2019-09-12T21:13:42.937Z",
+      "version": "Wzc0MzcsN10="
     },
     {
       "attributes": {
@@ -563,8 +563,8 @@
       },
       "references": [],
       "type": "visualization",
-      "updated_at": "2019-03-15T20:40:17.936Z",
-      "version": "WzkxNiw3XQ=="
+      "updated_at": "2019-09-12T21:13:42.937Z",
+      "version": "Wzc0MzgsN10="
     },
     {
       "attributes": {
@@ -601,7 +601,7 @@
                 "type": "list"
               },
               {
-                "fieldName": "aws.s3_request.bucket.name",
+                "fieldName": "aws.s3.bucket.name",
                 "id": "1549512142947",
                 "indexPattern": "metricbeat-*",
                 "label": "s3 bucket name",
@@ -630,8 +630,8 @@
       },
       "references": [],
       "type": "visualization",
-      "updated_at": "2019-03-15T20:47:17.880Z",
-      "version": "WzkyMCw3XQ=="
+      "updated_at": "2019-09-12T21:18:08.498Z",
+      "version": "Wzc3NTgsN10="
     },
     {
       "attributes": {
@@ -709,9 +709,9 @@
       },
       "references": [],
       "type": "visualization",
-      "updated_at": "2019-03-15T20:55:49.785Z",
-      "version": "WzkyMyw3XQ=="
+      "updated_at": "2019-09-12T21:13:42.937Z",
+      "version": "Wzc0NDAsN10="
     }
   ],
-  "version": "7.0.0-beta1"
+  "version": "7.4.0"
 }
diff --git a/x-pack/metricbeat/module/aws/fields.go b/x-pack/metricbeat/module/aws/fields.go
diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/data.json b/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/data.json
@@ -1,14 +1,18 @@
 {
     "@timestamp": "2017-10-12T08:05:34.853Z",
     "aws": {
+        "s3": {
+            "bucket": {
+                "name": "test-s3-ks-2"
+            }
+        },
         "s3_daily_storage": {
             "bucket": {
-                "name": "test-s3-ks-2",
                 "size": {
-                    "bytes": 256
+                    "bytes": 51665
                 }
             },
-            "number_of_objects": 8
+            "number_of_objects": 51
         }
     },
     "cloud": {
@@ -21,7 +25,8 @@
         "module": "aws"
     },
     "metricset": {
-        "name": "s3_daily_storage"
+        "name": "s3_daily_storage",
+        "period": 10000
     },
     "service": {
         "type": "aws"

diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/fields.yml b/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/fields.yml
@@ -4,10 +4,6 @@
     `s3_daily_storage` contains the daily storage metrics that were scraped from AWS CloudWatch which contains monitoring metrics sent by AWS S3.
   release: beta
   fields:
-    - name: bucket.name
-      type: keyword
-      description: >
-        Name of a S3 bucket.
     - name: bucket.size.bytes
       type: long
       format: bytes

diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go b/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go
@@ -212,7 +212,7 @@ func createCloudWatchEvents(outputs []cloudwatch.MetricDataResult, regionName st
 		return
 	}
 
-	resultMetricSetFields.Put("bucket.name", bucketName)
 	event.MetricSetFields = resultMetricSetFields
+	event.RootFields.Put("aws.s3.bucket.name", bucketName)
 	return
 }
diff --git a/x-pack/metricbeat/module/aws/s3_request/_meta/data.json b/x-pack/metricbeat/module/aws/s3_request/_meta/data.json
@@ -1,31 +1,32 @@
 {
     "@timestamp": "2017-10-12T08:05:34.853Z",
     "aws": {
-        "s3_request": {
+        "s3": {
             "bucket": {
                 "name": "test-s3-ks-2"
-            },
+            }
+        },
+        "s3_request": {
             "downloaded": {
-                "bytes": 123653
+                "bytes": 22307
             },
             "errors": {
                 "4xx": 0,
                 "5xx": 0
             },
             "latency": {
-                "first_byte.ms": 7975,
-                "total_request.ms": 9005
+                "first_byte.ms": 851,
+                "total_request.ms": 1146
             },
             "requests": {
-                "delete": 2,
-                "get": 80,
-                "head": 3,
-                "list": 37,
-                "put": 34,
-                "total": 156
+                "get": 2,
+                "head": 1,
+                "list": 2,
+                "put": 3,
+                "total": 8
             },
             "uploaded": {
-                "bytes": 2644
+                "bytes": 5557
             }
         }
     },
@@ -39,7 +40,8 @@
         "module": "aws"
     },
     "metricset": {
-        "name": "s3_request"
+        "name": "s3_request",
+        "period": 10000
     },
     "service": {
         "type": "aws"

diff --git a/x-pack/metricbeat/module/aws/s3_request/_meta/fields.yml b/x-pack/metricbeat/module/aws/s3_request/_meta/fields.yml
@@ -4,10 +4,6 @@
     `s3_request` contains request metrics that were scraped from AWS CloudWatch which contains monitoring metrics sent by AWS S3.
   release: beta
   fields:
-    - name: bucket.name
-      type: keyword
-      description: >
-        Name of a S3 bucket.
     - name: requests.total
       type: long
       description: >

diff --git a/x-pack/metricbeat/module/aws/s3_request/s3_request.go b/x-pack/metricbeat/module/aws/s3_request/s3_request.go
@@ -214,7 +214,7 @@ func createS3RequestEvents(outputs []cloudwatch.MetricDataResult, regionName str
 		return
 	}
 
-	resultMetricSetFields.Put("bucket.name", bucketName)
 	event.MetricSetFields = resultMetricSetFields
+	event.RootFields.Put("aws.s3.bucket.name", bucketName)
 	return
 }