diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index b7ea2c115c2..1171c8a3f21 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -238,6 +238,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Drop aws.vpcflow.pkt_srcaddr and aws.vpcflow.pkt_dstaddr when equal to "-". {pull}22721[22721] {issue}22716[22716] - Fix cisco umbrella module config by adding input variable. {pull}22892[22892] - Fix network.direction logic in zeek connection fileset. {pull}22967[22967] +- Fix aws s3 overview dashboard. {pull}23045[23045] - Fix bad `network.direction` values in Fortinet/firewall fileset. {pull}23072[23072] *Heartbeat* diff --git a/x-pack/metricbeat/module/aws/_meta/kibana/7/dashboard/Metricbeat-aws-s3-overview.json b/x-pack/metricbeat/module/aws/_meta/kibana/7/dashboard/Metricbeat-aws-s3-overview.json index e50f0e323c2..095ba9c700c 100644 --- a/x-pack/metricbeat/module/aws/_meta/kibana/7/dashboard/Metricbeat-aws-s3-overview.json +++ b/x-pack/metricbeat/module/aws/_meta/kibana/7/dashboard/Metricbeat-aws-s3-overview.json @@ -1,756 +1,766 @@ { - "objects": [ - { - "attributes": { - "description": "Overview of AWS S3 Metrics", - "hits": 0, - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "query": { - "language": "kuery", - "query": "" - } - } - }, - "optionsJSON": { - "hidePanelTitles": false, - "useMargins": true - }, - "panelsJSON": [ - { - "embeddableConfig": { - "title": "S3 Daily Storage Bucket Size in Bytes" - }, - "gridData": { - "h": 7, - "i": "1", - "w": 24, - "x": 0, - "y": 6 - }, - "panelIndex": "1", - "panelRefName": "panel_0", - "title": "S3 Daily Storage Bucket Size in Bytes", - "version": "7.7.0" - }, - { - "embeddableConfig": { - "title": "S3 Daily Storage Number of Objects" - }, - "gridData": { - "h": 7, - "i": "2", - "w": 24, - "x": 24, - "y": 6 - }, - "panelIndex": "2", - "panelRefName": "panel_1", - "title": "S3 Daily Storage Number of Objects", - "version": "7.7.0" - }, - { - "embeddableConfig": { - "title": "S3 Request Latency Total Request in ms" - }, - "gridData": { - "h": 7, - "i": "3", - "w": 24, - "x": 0, - "y": 13 - }, - "panelIndex": "3", - "panelRefName": "panel_2", - "title": "S3 Request Latency Total Request in ms", - "version": "7.7.0" - }, - { - "embeddableConfig": { - "title": "S3 Total Error 4xx" - }, - "gridData": { - "h": 6, - "i": "4", - "w": 13, - "x": 24, - "y": 0 - }, - "panelIndex": "4", - "panelRefName": "panel_3", - "title": "S3 Total Error 4xx", - "version": "7.7.0" - }, - { - "embeddableConfig": { - "title": "S3 Total Error 5xx" - }, - "gridData": { - "h": 6, - "i": "5", - "w": 11, - "x": 37, - "y": 0 - }, - "panelIndex": "5", - "panelRefName": "panel_4", - "title": "S3 Total Error 5xx", - "version": "7.7.0" - }, - { - "embeddableConfig": { - "title": "S3 Filters" - }, - "gridData": { - "h": 6, - "i": "6", - "w": 24, - "x": 0, - "y": 0 - }, - "panelIndex": "6", - "panelRefName": "panel_5", - "title": "S3 Filters", - "version": "7.7.0" - }, - { - "embeddableConfig": { - "title": "S3 Total Requests" + "objects": [ + { + "attributes": { + "description": "Overview of AWS S3 Metrics", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "title": "S3 Daily Storage Bucket Size in Bytes" + }, + "gridData": { + "h": 7, + "i": "1", + "w": 24, + "x": 0, + "y": 6 + }, + "panelIndex": "1", + "panelRefName": "panel_0", + "title": "S3 Daily Storage Bucket Size in Bytes", + "version": "7.7.0" + }, + { + "embeddableConfig": { + "title": "S3 Daily Storage Number of Objects" + }, + "gridData": { + "h": 7, + "i": "2", + "w": 24, + "x": 24, + "y": 6 + }, + "panelIndex": "2", + "panelRefName": "panel_1", + "title": "S3 Daily Storage Number of Objects", + "version": "7.7.0" + }, + { + "embeddableConfig": { + "title": "S3 Request Latency Total Request in ms" + }, + "gridData": { + "h": 7, + "i": "3", + "w": 24, + "x": 0, + "y": 13 + }, + "panelIndex": "3", + "panelRefName": "panel_2", + "title": "S3 Request Latency Total Request in ms", + "version": "7.7.0" + }, + { + "embeddableConfig": { + "title": "S3 Total Error 4xx" + }, + "gridData": { + "h": 6, + "i": "4", + "w": 13, + "x": 24, + "y": 0 + }, + "panelIndex": "4", + "panelRefName": "panel_3", + "title": "S3 Total Error 4xx", + "version": "7.7.0" + }, + { + "embeddableConfig": { + "title": "S3 Total Error 5xx" + }, + "gridData": { + "h": 6, + "i": "5", + "w": 11, + "x": 37, + "y": 0 + }, + "panelIndex": "5", + "panelRefName": "panel_4", + "title": "S3 Total Error 5xx", + "version": "7.7.0" + }, + { + "embeddableConfig": { + "title": "S3 Filters" + }, + "gridData": { + "h": 6, + "i": "6", + "w": 24, + "x": 0, + "y": 0 + }, + "panelIndex": "6", + "panelRefName": "panel_5", + "title": "S3 Filters", + "version": "7.7.0" + }, + { + "embeddableConfig": { + "title": "S3 Total Requests" + }, + "gridData": { + "h": 7, + "i": "7", + "w": 24, + "x": 24, + "y": 13 + }, + "panelIndex": "7", + "panelRefName": "panel_6", + "title": "S3 Total Requests", + "version": "7.7.0" + } + ], + "refreshInterval": { + "pause": true, + "value": 0 + }, + "timeFrom": "now-1d", + "timeRestore": true, + "timeTo": "now", + "title": "[Metricbeat AWS] S3 Overview", + "version": 1 }, - "gridData": { - "h": 7, - "i": "7", - "w": 24, - "x": 24, - "y": 13 + "id": "a096b830-4762-11e9-8062-c98a86cb6f94", + "migrationVersion": { + "dashboard": "7.4.2" }, - "panelIndex": "7", - "panelRefName": "panel_6", - "title": "S3 Total Requests", - "version": "7.7.0" - } - ], - "refreshInterval": { - "pause": true, - "value": 0 - }, - "timeFrom": "now-1d", - "timeRestore": true, - "timeTo": "now", - "title": "[Metricbeat AWS] S3 Overview", - "version": 1 - }, - "id": "a096b830-4762-11e9-8062-c98a86cb6f94", - "migrationVersion": { - "dashboard": "7.3.0" - }, - "references": [ - { - "id": "2dbb8f90-4760-11e9-8062-c98a86cb6f94", - "name": "panel_0", - "type": "visualization" - }, - { - "id": "3a3914d0-4761-11e9-8062-c98a86cb6f94", - "name": "panel_1", - "type": "visualization" - }, - { - "id": "2b2d58b0-4762-11e9-8062-c98a86cb6f94", - "name": "panel_2", - "type": "visualization" - }, - { - "id": "81d83c70-4762-11e9-8062-c98a86cb6f94", - "name": "panel_3", - "type": "visualization" - }, - { - "id": "8b34a100-4762-11e9-8062-c98a86cb6f94", - "name": "panel_4", - "type": "visualization" - }, - { - "id": "6e3285d0-4763-11e9-8062-c98a86cb6f94", - "name": "panel_5", - "type": "visualization" - }, - { - "id": "d186fd50-4763-11e9-8062-c98a86cb6f94", - "name": "panel_6", - "type": "visualization" - } - ], - "type": "dashboard", - "updated_at": "2020-04-06T16:29:50.517Z", - "version": "WzkxOSwyXQ==" - }, - { - "attributes": { - "description": "", - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "query": { - "language": "kuery", - "query": "" - } - } - }, - "title": "S3 Daily Storage Bucket Size in Bytes [Metricbeat AWS]", - "uiStateJSON": {}, - "version": 1, - "visState": { - "aggs": [], - "params": { - "axis_formatter": "number", - "axis_position": "left", - "axis_scale": "normal", - "background_color_rules": [ - { - "id": "f679e680-475f-11e9-a9de-e776805ecfc9" - } - ], - "bar_color_rules": [ - { - "id": "f703aff0-475f-11e9-a9de-e776805ecfc9" - } + "namespaces": [ + "default" ], - "default_index_pattern": "metricbeat-*", - "default_timefield": "@timestamp", - "drop_last_bucket": 0, - "gauge_color_rules": [ - { - "id": "f8388670-475f-11e9-a9de-e776805ecfc9" - } - ], - "gauge_inner_width": 10, - "gauge_style": "half", - "gauge_width": 10, - "id": "61ca57f0-469d-11e7-af02-69e470af7417", - "index_pattern": "metricbeat-*", - "interval": "1d", - "isModelInvalid": false, - "series": [ - { - "axis_position": "right", - "chart_type": "line", - "color": "#68BC00", - "fill": "0", - "formatter": "bytes", - "id": "61ca57f1-469d-11e7-af02-69e470af7417", - "line_width": 1, - "metrics": [ - { - "field": "aws.s3_daily_storage.bucket.size.bytes", - "id": "61ca57f2-469d-11e7-af02-69e470af7417", - "type": "avg" - } - ], - "point_size": 1, - "separate_axis": 0, - "split_mode": "terms", - "stacked": "none", - "terms_field": "aws.s3.bucket.name", - "terms_order_by": "61ca57f2-469d-11e7-af02-69e470af7417", - "terms_size": "5" - } + "references": [ + { + "id": "2dbb8f90-4760-11e9-8062-c98a86cb6f94", + "name": "panel_0", + "type": "visualization" + }, + { + "id": "3a3914d0-4761-11e9-8062-c98a86cb6f94", + "name": "panel_1", + "type": "visualization" + }, + { + "id": "2b2d58b0-4762-11e9-8062-c98a86cb6f94", + "name": "panel_2", + "type": "visualization" + }, + { + "id": "81d83c70-4762-11e9-8062-c98a86cb6f94", + "name": "panel_3", + "type": "visualization" + }, + { + "id": "8b34a100-4762-11e9-8062-c98a86cb6f94", + "name": "panel_4", + "type": "visualization" + }, + { + "id": "6e3285d0-4763-11e9-8062-c98a86cb6f94", + "name": "panel_5", + "type": "visualization" + }, + { + "id": "d186fd50-4763-11e9-8062-c98a86cb6f94", + "name": "panel_6", + "type": "visualization" + } ], - "show_grid": 1, - "show_legend": 1, - "time_field": "@timestamp", - "type": "top_n" - }, - "title": "S3 Daily Storage Bucket Size in Bytes [Metricbeat AWS]", - "type": "metrics" - } - }, - "id": "2dbb8f90-4760-11e9-8062-c98a86cb6f94", - "migrationVersion": { - "visualization": "7.4.2" - }, - "references": [], - "type": "visualization", - "updated_at": "2020-04-06T16:01:45.966Z", - "version": "Wzg2NCwyXQ==" - }, - { - "attributes": { - "description": "", - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "query": { - "language": "kuery", - "query": "" - } - } + "type": "dashboard", + "updated_at": "2020-12-08T17:28:15.675Z", + "version": "WzEzOTI3MCwxXQ==" }, - "title": "S3 Daily Storage Number of Objects [Metricbeat AWS]", - "uiStateJSON": {}, - "version": 1, - "visState": { - "aggs": [], - "params": { - "axis_formatter": "number", - "axis_position": "left", - "axis_scale": "normal", - "background_color_rules": [ - { - "id": "167ea870-4761-11e9-bf81-69a4e579cab5" - } - ], - "bar_color_rules": [ - { - "id": "01dad830-4761-11e9-bf81-69a4e579cab5" - } - ], - "default_index_pattern": "metricbeat-*", - "default_timefield": "@timestamp", - "drop_last_bucket": 0, - "id": "61ca57f0-469d-11e7-af02-69e470af7417", - "index_pattern": "metricbeat-*", - "interval": "1d", - "isModelInvalid": false, - "series": [ - { - "axis_position": "right", - "chart_type": "line", - "color": "#68BC00", - "fill": 0.5, - "formatter": "number", - "id": "61ca57f1-469d-11e7-af02-69e470af7417", - "line_width": 1, - "metrics": [ - { - "field": "aws.s3_daily_storage.number_of_objects", - "id": "61ca57f2-469d-11e7-af02-69e470af7417", - "type": "avg" - } - ], - "point_size": 1, - "separate_axis": 0, - "split_mode": "terms", - "stacked": "none", - "terms_field": "aws.s3.bucket.name", - "terms_order_by": "61ca57f2-469d-11e7-af02-69e470af7417", - "terms_size": "5" - } + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "S3 Daily Storage Bucket Size in Bytes [Metricbeat AWS]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "background_color_rules": [ + { + "id": "f679e680-475f-11e9-a9de-e776805ecfc9" + } + ], + "bar_color_rules": [ + { + "id": "f703aff0-475f-11e9-a9de-e776805ecfc9" + } + ], + "default_index_pattern": "metricbeat-*", + "default_timefield": "@timestamp", + "drop_last_bucket": 0, + "gauge_color_rules": [ + { + "id": "f8388670-475f-11e9-a9de-e776805ecfc9" + } + ], + "gauge_inner_width": 10, + "gauge_style": "half", + "gauge_width": 10, + "id": "61ca57f0-469d-11e7-af02-69e470af7417", + "index_pattern": "metricbeat-*", + "interval": "\u003e=1d", + "isModelInvalid": false, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": "0", + "formatter": "bytes", + "id": "61ca57f1-469d-11e7-af02-69e470af7417", + "line_width": 1, + "metrics": [ + { + "field": "aws.s3_daily_storage.bucket.size.bytes", + "id": "61ca57f2-469d-11e7-af02-69e470af7417", + "type": "avg" + } + ], + "point_size": 1, + "separate_axis": 0, + "split_color_mode": "gradient", + "split_mode": "terms", + "stacked": "none", + "terms_field": "aws.s3.bucket.name", + "terms_order_by": "61ca57f2-469d-11e7-af02-69e470af7417", + "terms_size": "20" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "@timestamp", + "time_range_mode": "entire_time_range", + "tooltip_mode": "show_all", + "type": "top_n" + }, + "title": "S3 Daily Storage Bucket Size in Bytes [Metricbeat AWS]", + "type": "metrics" + } + }, + "id": "2dbb8f90-4760-11e9-8062-c98a86cb6f94", + "migrationVersion": { + "visualization": "7.4.2" + }, + "namespaces": [ + "default" ], - "show_grid": 1, - "show_legend": 1, - "time_field": "@timestamp", - "type": "top_n" - }, - "title": "S3 Daily Storage Number of Objects [Metricbeat AWS]", - "type": "metrics" - } - }, - "id": "3a3914d0-4761-11e9-8062-c98a86cb6f94", - "migrationVersion": { - "visualization": "7.4.2" - }, - "references": [], - "type": "visualization", - "updated_at": "2020-04-06T16:15:51.106Z", - "version": "WzkwMywyXQ==" - }, - { - "attributes": { - "description": "", - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "query": { - "language": "kuery", - "query": "" - } - } + "references": [], + "type": "visualization", + "updated_at": "2020-12-09T20:59:44.051Z", + "version": "WzE0MDEyMSwxXQ==" }, - "title": "S3 Request Latency Total Request in ms [Metricbeat AWS]", - "uiStateJSON": {}, - "version": 1, - "visState": { - "aggs": [], - "params": { - "axis_formatter": "number", - "axis_position": "left", - "axis_scale": "normal", - "background_color_rules": [ - { - "id": "c0d11b00-4761-11e9-bf81-69a4e579cab5" - } - ], - "bar_color_rules": [ - { - "id": "67cb0930-4761-11e9-bf81-69a4e579cab5" - } - ], - "default_index_pattern": "metricbeat-*", - "default_timefield": "@timestamp", - "drop_last_bucket": 0, - "gauge_color_rules": [ - { - "id": "6eafde10-4761-11e9-bf81-69a4e579cab5" - } - ], - "gauge_inner_width": 10, - "gauge_style": "half", - "gauge_width": 10, - "id": "61ca57f0-469d-11e7-af02-69e470af7417", - "index_pattern": "metricbeat-*", - "interval": "1d", - "isModelInvalid": false, - "pivot_id": "aws.s3.bucket.name", - "series": [ - { - "axis_position": "right", - "chart_type": "line", - "color": "#68BC00", - "color_rules": [ - { - "id": "ac2ef870-4761-11e9-bf81-69a4e579cab5" - } - ], - "fill": 0.5, - "formatter": "number", - "id": "61ca57f1-469d-11e7-af02-69e470af7417", - "label": "Latency in ms", - "line_width": 1, - "metrics": [ - { - "field": "aws.s3_request.latency.total_request.ms", - "id": "61ca57f2-469d-11e7-af02-69e470af7417", - "type": "avg" - } - ], - "point_size": 1, - "separate_axis": 0, - "split_mode": "terms", - "stacked": "none", - "terms_field": "aws.s3.bucket.name", - "terms_order_by": "61ca57f2-469d-11e7-af02-69e470af7417", - "terms_size": "5" - } + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "S3 Daily Storage Number of Objects [Metricbeat AWS]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "background_color_rules": [ + { + "id": "167ea870-4761-11e9-bf81-69a4e579cab5" + } + ], + "bar_color_rules": [ + { + "id": "01dad830-4761-11e9-bf81-69a4e579cab5" + } + ], + "default_index_pattern": "metricbeat-*", + "default_timefield": "@timestamp", + "drop_last_bucket": 0, + "id": "61ca57f0-469d-11e7-af02-69e470af7417", + "index_pattern": "metricbeat-*", + "interval": "\u003e=1d", + "isModelInvalid": false, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "formatter": "number", + "id": "61ca57f1-469d-11e7-af02-69e470af7417", + "line_width": 1, + "metrics": [ + { + "field": "aws.s3_daily_storage.number_of_objects", + "id": "61ca57f2-469d-11e7-af02-69e470af7417", + "type": "avg" + } + ], + "point_size": 1, + "separate_axis": 0, + "split_color_mode": "gradient", + "split_mode": "terms", + "stacked": "none", + "terms_field": "aws.s3.bucket.name", + "terms_order_by": "61ca57f2-469d-11e7-af02-69e470af7417", + "terms_size": "20" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "@timestamp", + "time_range_mode": "entire_time_range", + "tooltip_mode": "show_all", + "type": "top_n" + }, + "title": "S3 Daily Storage Number of Objects [Metricbeat AWS]", + "type": "metrics" + } + }, + "id": "3a3914d0-4761-11e9-8062-c98a86cb6f94", + "migrationVersion": { + "visualization": "7.4.2" + }, + "namespaces": [ + "default" ], - "show_grid": 1, - "show_legend": 1, - "time_field": "@timestamp", - "type": "top_n" - }, - "title": "S3 Request Latency Total Request in ms [Metricbeat AWS]", - "type": "metrics" - } - }, - "id": "2b2d58b0-4762-11e9-8062-c98a86cb6f94", - "migrationVersion": { - "visualization": "7.4.2" - }, - "references": [], - "type": "visualization", - "updated_at": "2020-04-06T16:24:42.835Z", - "version": "WzkxMiwyXQ==" - }, - { - "attributes": { - "description": "", - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "query": { - "language": "kuery", - "query": "" - } - } + "references": [], + "type": "visualization", + "updated_at": "2020-12-09T21:08:43.773Z", + "version": "WzE0MDE0NiwxXQ==" }, - "title": "S3 Total Error 4xx [Metricbeat AWS]", - "uiStateJSON": {}, - "version": 1, - "visState": { - "aggs": [], - "params": { - "axis_formatter": "number", - "axis_position": "left", - "axis_scale": "normal", - "background_color_rules": [ - { - "id": "59207fe0-4762-11e9-bf81-69a4e579cab5" - } - ], - "bar_color_rules": [ - { - "id": "5ad9a190-4762-11e9-bf81-69a4e579cab5" - } - ], - "id": "61ca57f0-469d-11e7-af02-69e470af7417", - "index_pattern": "metricbeat-*", - "interval": "auto", - "series": [ - { - "axis_position": "right", - "chart_type": "line", - "color": "#68BC00", - "fill": 0.5, - "formatter": "number", - "id": "61ca57f1-469d-11e7-af02-69e470af7417", - "label": "Total # of HTTP 4xx Errors", - "line_width": 1, - "metrics": [ - { - "field": "aws.s3_request.errors.4xx", - "id": "61ca57f2-469d-11e7-af02-69e470af7417", - "type": "sum" - } - ], - "point_size": 1, - "separate_axis": 0, - "split_mode": "everything", - "stacked": "none" - } + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": {} + }, + "title": "S3 Request Latency Total Request in ms [Metricbeat AWS]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "background_color_rules": [ + { + "id": "c0d11b00-4761-11e9-bf81-69a4e579cab5" + } + ], + "bar_color_rules": [ + { + "id": "67cb0930-4761-11e9-bf81-69a4e579cab5" + } + ], + "default_index_pattern": "metricbeat-*", + "default_timefield": "@timestamp", + "drop_last_bucket": 0, + "gauge_color_rules": [ + { + "id": "6eafde10-4761-11e9-bf81-69a4e579cab5" + } + ], + "gauge_inner_width": 10, + "gauge_style": "half", + "gauge_width": 10, + "id": "61ca57f0-469d-11e7-af02-69e470af7417", + "index_pattern": "metricbeat-*", + "interval": "1d", + "isModelInvalid": false, + "pivot_id": "aws.s3.bucket.name", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "color_rules": [ + { + "id": "ac2ef870-4761-11e9-bf81-69a4e579cab5" + } + ], + "fill": 0.5, + "formatter": "number", + "id": "61ca57f1-469d-11e7-af02-69e470af7417", + "label": "Latency in ms", + "line_width": 1, + "metrics": [ + { + "field": "aws.s3_request.latency.total_request.ms", + "id": "61ca57f2-469d-11e7-af02-69e470af7417", + "type": "avg" + } + ], + "point_size": 1, + "separate_axis": 0, + "split_color_mode": "gradient", + "split_mode": "terms", + "stacked": "none", + "terms_field": "aws.s3.bucket.name", + "terms_order_by": "61ca57f2-469d-11e7-af02-69e470af7417", + "terms_size": "5" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "@timestamp", + "type": "top_n" + }, + "title": "S3 Request Latency Total Request in ms [Metricbeat AWS]", + "type": "metrics" + } + }, + "id": "2b2d58b0-4762-11e9-8062-c98a86cb6f94", + "migrationVersion": { + "visualization": "7.4.2" + }, + "namespaces": [ + "default" ], - "show_grid": 1, - "show_legend": 1, - "time_field": "@timestamp", - "type": "metric" - }, - "title": "AWS S3 Total Error 4xx", - "type": "metrics" - } - }, - "id": "81d83c70-4762-11e9-8062-c98a86cb6f94", - "migrationVersion": { - "visualization": "7.4.2" - }, - "references": [], - "type": "visualization", - "updated_at": "2020-04-03T21:29:11.214Z", - "version": "WzIwOCwyXQ==" - }, - { - "attributes": { - "description": "", - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "query": { - "language": "kuery", - "query": "" - } - } + "references": [], + "type": "visualization", + "updated_at": "2020-12-08T17:28:15.675Z", + "version": "WzEzOTI3MywxXQ==" }, - "title": "S3 Total Error 5xx [Metricbeat AWS]", - "uiStateJSON": {}, - "version": 1, - "visState": { - "aggs": [], - "params": { - "axis_formatter": "number", - "axis_position": "left", - "axis_scale": "normal", - "background_color_rules": [ - { - "id": "59207fe0-4762-11e9-bf81-69a4e579cab5" - } - ], - "bar_color_rules": [ - { - "id": "5ad9a190-4762-11e9-bf81-69a4e579cab5" - } - ], - "id": "61ca57f0-469d-11e7-af02-69e470af7417", - "index_pattern": "metricbeat-*", - "interval": "auto", - "series": [ - { - "axis_position": "right", - "chart_type": "line", - "color": "#68BC00", - "fill": 0.5, - "formatter": "number", - "id": "61ca57f1-469d-11e7-af02-69e470af7417", - "label": "Total # of HTTP 5xx Errors", - "line_width": 1, - "metrics": [ - { - "field": "aws.s3_request.errors.5xx", - "id": "61ca57f2-469d-11e7-af02-69e470af7417", - "type": "sum" - } - ], - "point_size": 1, - "separate_axis": 0, - "split_mode": "everything", - "stacked": "none" - } + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": {} + }, + "title": "S3 Total Error 4xx [Metricbeat AWS]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "background_color_rules": [ + { + "id": "59207fe0-4762-11e9-bf81-69a4e579cab5" + } + ], + "bar_color_rules": [ + { + "id": "5ad9a190-4762-11e9-bf81-69a4e579cab5" + } + ], + "id": "61ca57f0-469d-11e7-af02-69e470af7417", + "index_pattern": "metricbeat-*", + "interval": "auto", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "formatter": "number", + "id": "61ca57f1-469d-11e7-af02-69e470af7417", + "label": "Total # of HTTP 4xx Errors", + "line_width": 1, + "metrics": [ + { + "field": "aws.s3_request.errors.4xx", + "id": "61ca57f2-469d-11e7-af02-69e470af7417", + "type": "sum" + } + ], + "point_size": 1, + "separate_axis": 0, + "split_color_mode": "gradient", + "split_mode": "everything", + "stacked": "none" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "@timestamp", + "type": "metric" + }, + "title": "AWS S3 Total Error 4xx", + "type": "metrics" + } + }, + "id": "81d83c70-4762-11e9-8062-c98a86cb6f94", + "migrationVersion": { + "visualization": "7.4.2" + }, + "namespaces": [ + "default" ], - "show_grid": 1, - "show_legend": 1, - "time_field": "@timestamp", - "type": "metric" - }, - "title": "AWS S3 Total Error 5xx", - "type": "metrics" - } - }, - "id": "8b34a100-4762-11e9-8062-c98a86cb6f94", - "migrationVersion": { - "visualization": "7.4.2" - }, - "references": [], - "type": "visualization", - "updated_at": "2020-04-03T21:29:11.214Z", - "version": "WzIwOSwyXQ==" - }, - { - "attributes": { - "description": "", - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "query": { - "language": "kuery", - "query": "" - } - } + "references": [], + "type": "visualization", + "updated_at": "2020-12-08T17:28:15.675Z", + "version": "WzEzOTI3NCwxXQ==" }, - "title": "S3 Filters [Metricbeat AWS]", - "uiStateJSON": {}, - "version": 1, - "visState": { - "aggs": [], - "params": { - "controls": [ - { - "fieldName": "cloud.region", - "id": "1549397251041", - "indexPattern": "metricbeat-*", - "label": "region", - "options": { - "dynamicOptions": true, - "multiselect": true, - "order": "desc", - "size": 5, - "type": "terms" - }, - "parent": "", - "type": "list" - }, - { - "fieldName": "aws.s3.bucket.name", - "id": "1549512142947", - "indexPattern": "metricbeat-*", - "label": "s3 bucket name", - "options": { - "dynamicOptions": true, - "multiselect": true, - "order": "desc", - "size": 5, - "type": "terms" + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": {} }, - "parent": "", - "type": "list" - } + "title": "S3 Total Error 5xx [Metricbeat AWS]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "background_color_rules": [ + { + "id": "59207fe0-4762-11e9-bf81-69a4e579cab5" + } + ], + "bar_color_rules": [ + { + "id": "5ad9a190-4762-11e9-bf81-69a4e579cab5" + } + ], + "id": "61ca57f0-469d-11e7-af02-69e470af7417", + "index_pattern": "metricbeat-*", + "interval": "auto", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "formatter": "number", + "id": "61ca57f1-469d-11e7-af02-69e470af7417", + "label": "Total # of HTTP 5xx Errors", + "line_width": 1, + "metrics": [ + { + "field": "aws.s3_request.errors.5xx", + "id": "61ca57f2-469d-11e7-af02-69e470af7417", + "type": "sum" + } + ], + "point_size": 1, + "separate_axis": 0, + "split_color_mode": "gradient", + "split_mode": "everything", + "stacked": "none" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "@timestamp", + "type": "metric" + }, + "title": "AWS S3 Total Error 5xx", + "type": "metrics" + } + }, + "id": "8b34a100-4762-11e9-8062-c98a86cb6f94", + "migrationVersion": { + "visualization": "7.4.2" + }, + "namespaces": [ + "default" ], - "pinFilters": false, - "updateFiltersOnChange": true, - "useTimeFilter": false - }, - "title": "AWS S3 Filters", - "type": "input_control_vis" - } - }, - "id": "6e3285d0-4763-11e9-8062-c98a86cb6f94", - "migrationVersion": { - "visualization": "7.4.2" - }, - "references": [], - "type": "visualization", - "updated_at": "2020-04-03T21:29:11.214Z", - "version": "WzIxMCwyXQ==" - }, - { - "attributes": { - "description": "", - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "query": { - "language": "kuery", - "query": "" - } - } + "references": [], + "type": "visualization", + "updated_at": "2020-12-08T17:28:15.675Z", + "version": "WzEzOTI3NSwxXQ==" }, - "title": "S3 Total Requests [Metricbeat AWS]", - "uiStateJSON": {}, - "version": 1, - "visState": { - "aggs": [], - "params": { - "axis_formatter": "number", - "axis_position": "left", - "axis_scale": "normal", - "background_color_rules": [ - { - "id": "c03c4320-4763-11e9-b811-fd5d24a641d7" - } - ], - "bar_color_rules": [ - { - "id": "c7b9fca0-4763-11e9-b811-fd5d24a641d7" - } + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "S3 Filters [Metricbeat AWS]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "controls": [ + { + "fieldName": "cloud.region", + "id": "1549397251041", + "indexPattern": "metricbeat-*", + "label": "region", + "options": { + "dynamicOptions": true, + "multiselect": true, + "order": "desc", + "size": 5, + "type": "terms" + }, + "parent": "", + "type": "list" + }, + { + "fieldName": "aws.s3.bucket.name", + "id": "1549512142947", + "indexPattern": "metricbeat-*", + "label": "s3 bucket name", + "options": { + "dynamicOptions": true, + "multiselect": true, + "order": "desc", + "size": 5, + "type": "terms" + }, + "parent": "", + "type": "list" + } + ], + "pinFilters": false, + "updateFiltersOnChange": true, + "useTimeFilter": false + }, + "title": "AWS S3 Filters", + "type": "input_control_vis" + } + }, + "id": "6e3285d0-4763-11e9-8062-c98a86cb6f94", + "migrationVersion": { + "visualization": "7.4.2" + }, + "namespaces": [ + "default" ], - "default_index_pattern": "metricbeat-*", - "default_timefield": "@timestamp", - "drop_last_bucket": 0, - "id": "61ca57f0-469d-11e7-af02-69e470af7417", - "index_pattern": "metricbeat-*", - "interval": "1d", - "isModelInvalid": false, - "series": [ - { - "axis_position": "right", - "chart_type": "line", - "color": "#68BC00", - "fill": "0", - "formatter": "number", - "id": "61ca57f1-469d-11e7-af02-69e470af7417", - "line_width": 1, - "metrics": [ - { - "field": "aws.s3_request.requests.total", - "id": "61ca57f2-469d-11e7-af02-69e470af7417", - "type": "avg" - } - ], - "offset_time": "", - "point_size": 1, - "separate_axis": 0, - "split_mode": "terms", - "stacked": "none", - "terms_field": "aws.s3.bucket.name", - "terms_order_by": "61ca57f2-469d-11e7-af02-69e470af7417", - "terms_size": "5" - } + "references": [], + "type": "visualization", + "updated_at": "2020-12-08T17:28:15.675Z", + "version": "WzEzOTI3NiwxXQ==" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": {} + }, + "title": "S3 Total Requests [Metricbeat AWS]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "background_color_rules": [ + { + "id": "c03c4320-4763-11e9-b811-fd5d24a641d7" + } + ], + "bar_color_rules": [ + { + "id": "c7b9fca0-4763-11e9-b811-fd5d24a641d7" + } + ], + "default_index_pattern": "metricbeat-*", + "default_timefield": "@timestamp", + "drop_last_bucket": 0, + "id": "61ca57f0-469d-11e7-af02-69e470af7417", + "index_pattern": "metricbeat-*", + "interval": "1d", + "isModelInvalid": false, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": "0", + "formatter": "number", + "id": "61ca57f1-469d-11e7-af02-69e470af7417", + "line_width": 1, + "metrics": [ + { + "field": "aws.s3_request.requests.total", + "id": "61ca57f2-469d-11e7-af02-69e470af7417", + "type": "avg" + } + ], + "offset_time": "", + "point_size": 1, + "separate_axis": 0, + "split_color_mode": "gradient", + "split_mode": "terms", + "stacked": "none", + "terms_field": "aws.s3.bucket.name", + "terms_order_by": "61ca57f2-469d-11e7-af02-69e470af7417", + "terms_size": "5" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "@timestamp", + "type": "top_n" + }, + "title": "S3 Total Requests [Metricbeat AWS]", + "type": "metrics" + } + }, + "id": "d186fd50-4763-11e9-8062-c98a86cb6f94", + "migrationVersion": { + "visualization": "7.4.2" + }, + "namespaces": [ + "default" ], - "show_grid": 1, - "show_legend": 1, - "time_field": "@timestamp", - "type": "top_n" - }, - "title": "S3 Total Requests [Metricbeat AWS]", - "type": "metrics" + "references": [], + "type": "visualization", + "updated_at": "2020-12-08T17:28:15.675Z", + "version": "WzEzOTI3NywxXQ==" } - }, - "id": "d186fd50-4763-11e9-8062-c98a86cb6f94", - "migrationVersion": { - "visualization": "7.4.2" - }, - "references": [], - "type": "visualization", - "updated_at": "2020-04-06T16:25:02.532Z", - "version": "WzkxMywyXQ==" - } - ], - "version": "7.7.0" + ], + "version": "7.10.0" } diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/data.json b/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/data.json index 4285889e5d4..5b1e2546545 100644 --- a/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/data.json +++ b/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/data.json @@ -5,13 +5,17 @@ "namespace": "AWS/S3" }, "dimensions": { - "BucketName": "filebeat-aws-elb-test", - "StorageType": "AllStorageTypes" + "StorageType": "StandardStorage" }, "s3": { - "metrics": { - "NumberOfObjects": { - "avg": 57828 + "bucket": { + "name": "filebeat-aws-elb-test" + } + }, + "s3_daily_storage": { + "bucket": { + "size": { + "bytes": 76353832 } } } diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/docs.asciidoc index 08283d1366d..a11ed790a20 100644 --- a/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/docs.asciidoc @@ -20,6 +20,10 @@ The aws s3_daily_storage metricset and s3_request metricset shares one predefine image::./images/metricbeat-aws-s3-overview.png[] +Note: If only `s3_daily_storage` metricset is enabled or s3 request metrics are +not enabled for the specific S3 bucket, some visualizations in this dashboard +will be empty. + [float] === Configuration example [source,yaml] diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/manifest.yml b/x-pack/metricbeat/module/aws/s3_daily_storage/manifest.yml index 83ed42545c0..5401ba81f41 100644 --- a/x-pack/metricbeat/module/aws/s3_daily_storage/manifest.yml +++ b/x-pack/metricbeat/module/aws/s3_daily_storage/manifest.yml @@ -20,3 +20,7 @@ processors: to: "aws.s3_daily_storage.bucket.size.bytes" - from: "aws.dimensions.BucketName" to: "aws.s3.bucket.name" + - drop_fields: + ignore_missing: true + fields: + - "aws.s3.metrics" diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage_integration_test.go b/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage_integration_test.go index a398926b462..e570cd5bb73 100644 --- a/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage_integration_test.go +++ b/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage_integration_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/assert" + _ "github.com/elastic/beats/v7/libbeat/processors/actions" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" "github.com/elastic/beats/v7/x-pack/metricbeat/module/aws/mtest" ) diff --git a/x-pack/metricbeat/module/aws/s3_request/_meta/data.json b/x-pack/metricbeat/module/aws/s3_request/_meta/data.json index 8bdda96ef8d..5780e8215b5 100644 --- a/x-pack/metricbeat/module/aws/s3_request/_meta/data.json +++ b/x-pack/metricbeat/module/aws/s3_request/_meta/data.json @@ -5,32 +5,32 @@ "namespace": "AWS/S3" }, "dimensions": { - "BucketName": "test-ks-1", - "FilterId": "EntireBucket" + "FilterId": "request-metrics" }, "s3": { - "metrics": { - "4xxErrors": { - "avg": 0 - }, - "5xxErrors": { - "avg": 0 - }, - "AllRequests": { - "avg": 1 - }, - "BytesUploaded": { - "avg": 684 - }, - "FirstByteLatency": { - "avg": 21.791666666666668 - }, - "PutRequests": { - "avg": 1 + "bucket": { + "name": "test-ks-1" + } + }, + "s3_request": { + "errors": { + "4xx": 0, + "5xx": 0 + }, + "latency": { + "first_byte": { + "ms": 47.25 }, - "TotalRequestLatency": { - "avg": 35.916666666666664 + "total_request": { + "ms": 57.75 } + }, + "requests": { + "put": 1, + "total": 1 + }, + "uploaded": { + "bytes": 463 } } }, diff --git a/x-pack/metricbeat/module/aws/s3_request/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/s3_request/_meta/docs.asciidoc index edde8a25c00..458ad006d22 100644 --- a/x-pack/metricbeat/module/aws/s3_request/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/s3_request/_meta/docs.asciidoc @@ -2,6 +2,13 @@ The s3_request metricset of aws module allows you to monitor your AWS S3 buckets fetches Cloudwatch daily storage metrics for each S3 bucket from https://docs.aws.amazon.com/AmazonS3/latest/dev/cloudwatch-monitoring.html[S3 CloudWatch Request Metrics for Buckets]. +Note: Request metrics are not enabled by default. You must opt into +request metrics by configuring them in the console or using the Amazon S3 API. +Please see +https://docs.aws.amazon.com/AmazonS3/latest/user-guide/configure-metrics.html[How to +Configure Request Metrics for S3] for instructions on how to enable request metrics for +each S3 bucket. + [float] === AWS Permissions Some specific AWS permissions are required for IAM user to collect AWS s3_request metrics. @@ -20,6 +27,9 @@ The aws s3_request metricset and s3_daily_storage metricset shares one predefine image::./images/metricbeat-aws-s3-overview.png[] +Note: If s3 request metrics are not enabled for the specific S3 bucket, some +visualizations in this dashboard will be empty. + [float] === Configuration example [source,yaml] diff --git a/x-pack/metricbeat/module/aws/s3_request/manifest.yml b/x-pack/metricbeat/module/aws/s3_request/manifest.yml index 6a401fed08f..2fc3335552a 100644 --- a/x-pack/metricbeat/module/aws/s3_request/manifest.yml +++ b/x-pack/metricbeat/module/aws/s3_request/manifest.yml @@ -62,3 +62,7 @@ processors: to: "aws.s3_request.latency.total_request.ms" - from: "aws.dimensions.BucketName" to: "aws.s3.bucket.name" + - drop_fields: + ignore_missing: true + fields: + - "aws.s3.metrics" diff --git a/x-pack/metricbeat/module/aws/s3_request/s3_request_integration_test.go b/x-pack/metricbeat/module/aws/s3_request/s3_request_integration_test.go index c7b37de2af3..6836d846975 100644 --- a/x-pack/metricbeat/module/aws/s3_request/s3_request_integration_test.go +++ b/x-pack/metricbeat/module/aws/s3_request/s3_request_integration_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/assert" + _ "github.com/elastic/beats/v7/libbeat/processors/actions" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" "github.com/elastic/beats/v7/x-pack/metricbeat/module/aws/mtest" )