Skip to content

Commit

Permalink
[AWS] Move transit gateway lightweight module config into integration (
Browse files Browse the repository at this point in the history
  • Loading branch information
zmoog authored Jul 27, 2022
1 parent eb82fe9 commit 972cfdc
Show file tree
Hide file tree
Showing 7 changed files with 170 additions and 72 deletions.
5 changes: 5 additions & 0 deletions packages/aws/changelog.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
# newer versions go on top
- version: "1.19.1"
changes:
- description: Move Transit Gateway metrics config from beats to integrations
type: enhancement
link: https://github.com/elastic/integrations/pull/3840
- version: "1.19.0"
changes:
- description: Add Kinesis metrics datastream
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
metricsets: ["transitgateway"]
metricsets: ["cloudwatch"]
period: {{period}}
{{#if access_key_id}}
access_key_id: {{access_key_id}}
Expand Down Expand Up @@ -35,4 +35,16 @@ tags_filter: {{tags_filter}}
{{/if}}
{{#if proxy_url }}
proxy_url: {{proxy_url}}
{{/if}}
{{/if}}
metrics:
- namespace: AWS/TransitGateway
statistic: ["Sum"]
name:
- BytesIn
- BytesOut
- PacketsIn
- PacketsOut
- PacketDropCountBlackhole
- PacketDropCountNoRoute
- BytesDropCountNoRoute
- BytesDropCountBlackhole
6 changes: 6 additions & 0 deletions packages/aws/data_stream/transitgateway/fields/fields.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,12 @@
- name: PacketDropCountNoRoute.sum
type: long
description: The number of packets dropped because they did not match a route.
- name: BytesDropCountNoRoute.sum
type: long
description: The number of bytes dropped because they did not match a route.
- name: BytesDropCountBlackhole.sum
type: long
description: The number of bytes dropped because they matched a blackhole route.
- name: cloudwatch
type: group
fields:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,3 @@
type: object
description: |
Metric dimensions.
- name: '*.metrics.*.*'
type: object
description: |
Metrics that returned from Cloudwatch API query.
103 changes: 71 additions & 32 deletions packages/aws/data_stream/transitgateway/sample_event.json
Original file line number Diff line number Diff line change
@@ -1,63 +1,102 @@
{
"@timestamp": "2020-05-28T20:10:20.953Z",
"agent": {
"name": "a20ad158868c",
"id": "ac8c5411-b1d9-486a-baf7-a719744b13e5",
"ephemeral_id": "d43b281f-9a3e-48be-a7b2-e70c0d0b9acd",
"type": "metricbeat",
"version": "8.1.0"
},
"elastic_agent": {
"id": "ac8c5411-b1d9-486a-baf7-a719744b13e5",
"version": "8.1.0",
"snapshot": false
},
"cloud": {
"provider": "aws",
"region": "us-west-2",
"region": "eu-west-1",
"account": {
"name": "elastic-beats",
"id": "428152502467"
"name": "elastic-observability",
"id": "627286350134"
}
},
"@timestamp": "2022-07-26T21:58:00.000Z",
"ecs": {
"version": "8.0.0"
},
"data_stream": {
"namespace": "default",
"type": "metrics",
"dataset": "aws.transitgateway"
},
"service": {
"type": "aws"
},
"host": {
"hostname": "a20ad158868c",
"os": {
"kernel": "5.10.104-linuxkit",
"codename": "focal",
"name": "Ubuntu",
"type": "linux",
"family": "debian",
"version": "20.04.3 LTS (Focal Fossa)",
"platform": "ubuntu"
},
"containerized": false,
"ip": [
"172.20.0.7"
],
"name": "a20ad158868c",
"mac": [
"02:42:ac:14:00:07"
],
"architecture": "aarch64"
},
"metricset": {
"period": 60000,
"name": "cloudwatch"
},
"aws": {
"cloudwatch": {
"namespace": "AWS/TransitGateway"
},
"transitgateway": {
"metrics": {
"PacketsIn": {
"PacketsOut": {
"sum": 0
},
"BytesIn": {
"BytesDropCountNoRoute": {
"sum": 0
},
"PacketDropCountNoRoute": {
"sum": 0
},
"BytesOut": {
"sum": 0
},
"PacketsOut": {
"BytesIn": {
"sum": 0
},
"PacketDropCountBlackhole": {
"PacketsIn": {
"sum": 0
},
"PacketDropCountNoRoute": {
"BytesDropCountBlackhole": {
"sum": 0
},
"PacketDropCountBlackhole": {
"sum": 0
}
}
},
"cloudwatch": {
"namespace": "AWS/TransitGateway"
},
"dimensions": {
"TransitGateway": "tgw-0630672a32f12808a"
"TransitGateway": "tgw-04653af6191a63891"
}
},
"ecs": {
"version": "1.5.0"
},
"agent": {
"id": "12f376ef-5186-4e8b-a175-70f1140a8f30",
"name": "MacBook-Elastic.local",
"type": "metricbeat",
"version": "8.0.0",
"ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b"
},
"event": {
"dataset": "aws.transitgateway",
"duration": 1614567042,
"agent_id_status": "verified",
"ingested": "2022-07-26T21:59:04Z",
"module": "aws",
"duration": 12762825681
},
"metricset": {
"period": 60000,
"name": "transitgateway"
},
"service": {
"type": "aws"
"dataset": "aws.transitgateway"
}
}
106 changes: 73 additions & 33 deletions packages/aws/docs/transitgateway.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,66 +6,105 @@ An example event for `transitgateway` looks as following:

```json
{
"@timestamp": "2020-05-28T20:10:20.953Z",
"agent": {
"name": "a20ad158868c",
"id": "ac8c5411-b1d9-486a-baf7-a719744b13e5",
"ephemeral_id": "d43b281f-9a3e-48be-a7b2-e70c0d0b9acd",
"type": "metricbeat",
"version": "8.1.0"
},
"elastic_agent": {
"id": "ac8c5411-b1d9-486a-baf7-a719744b13e5",
"version": "8.1.0",
"snapshot": false
},
"cloud": {
"provider": "aws",
"region": "us-west-2",
"region": "eu-west-1",
"account": {
"name": "elastic-beats",
"id": "428152502467"
"name": "elastic-observability",
"id": "627286350134"
}
},
"@timestamp": "2022-07-26T21:58:00.000Z",
"ecs": {
"version": "8.0.0"
},
"data_stream": {
"namespace": "default",
"type": "metrics",
"dataset": "aws.transitgateway"
},
"service": {
"type": "aws"
},
"host": {
"hostname": "a20ad158868c",
"os": {
"kernel": "5.10.104-linuxkit",
"codename": "focal",
"name": "Ubuntu",
"type": "linux",
"family": "debian",
"version": "20.04.3 LTS (Focal Fossa)",
"platform": "ubuntu"
},
"containerized": false,
"ip": [
"172.20.0.7"
],
"name": "a20ad158868c",
"mac": [
"02:42:ac:14:00:07"
],
"architecture": "aarch64"
},
"metricset": {
"period": 60000,
"name": "cloudwatch"
},
"aws": {
"cloudwatch": {
"namespace": "AWS/TransitGateway"
},
"transitgateway": {
"metrics": {
"PacketsIn": {
"PacketsOut": {
"sum": 0
},
"BytesIn": {
"BytesDropCountNoRoute": {
"sum": 0
},
"PacketDropCountNoRoute": {
"sum": 0
},
"BytesOut": {
"sum": 0
},
"PacketsOut": {
"BytesIn": {
"sum": 0
},
"PacketDropCountBlackhole": {
"PacketsIn": {
"sum": 0
},
"PacketDropCountNoRoute": {
"BytesDropCountBlackhole": {
"sum": 0
},
"PacketDropCountBlackhole": {
"sum": 0
}
}
},
"cloudwatch": {
"namespace": "AWS/TransitGateway"
},
"dimensions": {
"TransitGateway": "tgw-0630672a32f12808a"
"TransitGateway": "tgw-04653af6191a63891"
}
},
"ecs": {
"version": "1.5.0"
},
"agent": {
"id": "12f376ef-5186-4e8b-a175-70f1140a8f30",
"name": "MacBook-Elastic.local",
"type": "metricbeat",
"version": "8.0.0",
"ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b"
},
"event": {
"dataset": "aws.transitgateway",
"duration": 1614567042,
"agent_id_status": "verified",
"ingested": "2022-07-26T21:59:04Z",
"module": "aws",
"duration": 12762825681
},
"metricset": {
"period": 60000,
"name": "transitgateway"
},
"service": {
"type": "aws"
"dataset": "aws.transitgateway"
}
}
```
Expand All @@ -75,13 +114,14 @@ An example event for `transitgateway` looks as following:
| Field | Description | Type |
|---|---|---|
| @timestamp | Event timestamp. | date |
| aws.\*.metrics.\*.\* | Metrics that returned from Cloudwatch API query. | object |
| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword |
| aws.dimensions.\* | Metric dimensions. | object |
| aws.dimensions.TransitGateway | Filters the metric data by transit gateway. | keyword |
| aws.dimensions.TransitGatewayAttachment | Filters the metric data by transit gateway attachment. | keyword |
| aws.s3.bucket.name | Name of a S3 bucket. | keyword |
| aws.tags.\* | Tag key value pairs from aws resources. | object |
| aws.transitgateway.metrics.BytesDropCountBlackhole.sum | The number of bytes dropped because they matched a blackhole route. | long |
| aws.transitgateway.metrics.BytesDropCountNoRoute.sum | The number of bytes dropped because they did not match a route. | long |
| aws.transitgateway.metrics.BytesIn.sum | The number of bytes received by the transit gateway. | long |
| aws.transitgateway.metrics.BytesOut.sum | The number of bytes sent from the transit gateway. | long |
| aws.transitgateway.metrics.PacketDropCountBlackhole.sum | The number of packets dropped because they matched a blackhole route. | long |
Expand Down
2 changes: 1 addition & 1 deletion packages/aws/manifest.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
format_version: 1.0.0
name: aws
title: AWS
version: 1.19.0
version: 1.19.1
license: basic
description: Collect logs and metrics from Amazon Web Services with Elastic Agent.
type: integration
Expand Down

0 comments on commit 972cfdc

Please sign in to comment.