From 6a8d9a396af310a9a34e205fb390b7c29f85ab56 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 24 Jan 2022 11:03:56 -0700 Subject: [PATCH 1/4] Add data_stream.dataset option for custom aws-cloudwatch log input --- .../test-cloudwatch-ec2.log-expected.json | 84 ++++++++----------- .../agent/stream/aws-cloudwatch.yml.hbs | 3 + .../elasticsearch/ingest_pipeline/default.yml | 41 +++------ .../cloudwatch_logs/fields/ecs.yml | 4 + .../data_stream/cloudwatch_logs/manifest.yml | 7 ++ packages/aws/docs/cloudwatch.md | 2 + packages/aws/manifest.yml | 2 +- 7 files changed, 65 insertions(+), 78 deletions(-) diff --git a/packages/aws/data_stream/cloudwatch_logs/_dev/test/pipeline/test-cloudwatch-ec2.log-expected.json b/packages/aws/data_stream/cloudwatch_logs/_dev/test/pipeline/test-cloudwatch-ec2.log-expected.json index 20dcfcf756c..c544ea851d1 100644 --- a/packages/aws/data_stream/cloudwatch_logs/_dev/test/pipeline/test-cloudwatch-ec2.log-expected.json +++ b/packages/aws/data_stream/cloudwatch_logs/_dev/test/pipeline/test-cloudwatch-ec2.log-expected.json @@ -1,109 +1,97 @@ { "expected": [ { - "@timestamp": "2020-02-20T07:01:01.000Z", + "cloud": { + "provider": "aws" + }, "ecs": { - "version": "8.0.0" + "version": "1.12.0" }, + "message": "2020-02-20T07:01:01.000Z Feb 20 07:01:01 ip-172-31-81-156 systemd: Stopping User Slice of root.", "event": { - "ingested": "2022-01-09T23:41:38.962436254Z", + "kind": "event", "original": "2020-02-20T07:01:01.000Z Feb 20 07:01:01 ip-172-31-81-156 systemd: Stopping User Slice of root." }, - "aws": { - "cloudwatch": { - "message": "ip-172-31-81-156 systemd: Stopping User Slice of root." - } - }, "tags": [ "preserve_original_event" ] }, { - "@timestamp": "2020-02-20T07:02:18.000Z", + "cloud": { + "provider": "aws" + }, "ecs": { - "version": "8.0.0" + "version": "1.12.0" }, + "message": "2020-02-20T07:02:18.000Z Feb 20 07:02:18 ip-172-31-81-156 dhclient[3000]: XMT: Solicit on eth0, interval 125240ms.", "event": { - "ingested": "2022-01-09T23:41:38.962442522Z", + "kind": "event", "original": "2020-02-20T07:02:18.000Z Feb 20 07:02:18 ip-172-31-81-156 dhclient[3000]: XMT: Solicit on eth0, interval 125240ms." }, - "aws": { - "cloudwatch": { - "message": "ip-172-31-81-156 dhclient[3000]: XMT: Solicit on eth0, interval 125240ms." - } - }, "tags": [ "preserve_original_event" ] }, { - "@timestamp": "2020-02-20T07:02:37.000Z", + "cloud": { + "provider": "aws" + }, "ecs": { - "version": "8.0.0" + "version": "1.12.0" }, + "message": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: DHCPREQUEST on eth0 to 172.31.80.1 port 67 (xid=0x4575af22)", "event": { - "ingested": "2022-01-09T23:41:38.962444166Z", + "kind": "event", "original": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: DHCPREQUEST on eth0 to 172.31.80.1 port 67 (xid=0x4575af22)" }, - "aws": { - "cloudwatch": { - "message": "ip-172-31-81-156 dhclient[2898]: DHCPREQUEST on eth0 to 172.31.80.1 port 67 (xid=0x4575af22)" - } - }, "tags": [ "preserve_original_event" ] }, { - "@timestamp": "2020-02-20T07:02:37.000Z", + "cloud": { + "provider": "aws" + }, "ecs": { - "version": "8.0.0" + "version": "1.12.0" }, + "message": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: DHCPACK from 172.31.80.1 (xid=0x4575af22)", "event": { - "ingested": "2022-01-09T23:41:38.962445580Z", + "kind": "event", "original": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: DHCPACK from 172.31.80.1 (xid=0x4575af22)" }, - "aws": { - "cloudwatch": { - "message": "ip-172-31-81-156 dhclient[2898]: DHCPACK from 172.31.80.1 (xid=0x4575af22)" - } - }, "tags": [ "preserve_original_event" ] }, { - "@timestamp": "2020-02-20T07:02:37.000Z", + "cloud": { + "provider": "aws" + }, "ecs": { - "version": "8.0.0" + "version": "1.12.0" }, + "message": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: bound to 172.31.81.156 -- renewal in 1599 seconds.", "event": { - "ingested": "2022-01-09T23:41:38.962446977Z", + "kind": "event", "original": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: bound to 172.31.81.156 -- renewal in 1599 seconds." }, - "aws": { - "cloudwatch": { - "message": "ip-172-31-81-156 dhclient[2898]: bound to 172.31.81.156 -- renewal in 1599 seconds." - } - }, "tags": [ "preserve_original_event" ] }, { - "@timestamp": "2020-02-20T07:02:37.000Z", + "cloud": { + "provider": "aws" + }, "ecs": { - "version": "8.0.0" + "version": "1.12.0" }, + "message": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 ec2net: [get_meta] Trying to get http://169.254.169.254/latest/meta-data/network/interfaces/macs/12:e2:a9:95:8b:97/local-ipv4s", "event": { - "ingested": "2022-01-09T23:41:38.962448339Z", + "kind": "event", "original": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 ec2net: [get_meta] Trying to get http://169.254.169.254/latest/meta-data/network/interfaces/macs/12:e2:a9:95:8b:97/local-ipv4s" }, - "aws": { - "cloudwatch": { - "message": "ip-172-31-81-156 ec2net: [get_meta] Trying to get http://169.254.169.254/latest/meta-data/network/interfaces/macs/12:e2:a9:95:8b:97/local-ipv4s" - } - }, "tags": [ "preserve_original_event" ] diff --git a/packages/aws/data_stream/cloudwatch_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/cloudwatch_logs/agent/stream/aws-cloudwatch.yml.hbs index c1576eedae0..d5f72d845eb 100644 --- a/packages/aws/data_stream/cloudwatch_logs/agent/stream/aws-cloudwatch.yml.hbs +++ b/packages/aws/data_stream/cloudwatch_logs/agent/stream/aws-cloudwatch.yml.hbs @@ -1,3 +1,6 @@ +data_stream: +dataset: {{data_stream.dataset}} + {{#unless log_group_name}} {{#unless log_group_name_prefix}} {{#if log_group_arn }} diff --git a/packages/aws/data_stream/cloudwatch_logs/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/cloudwatch_logs/elasticsearch/ingest_pipeline/default.yml index c1296b33753..f61c3d671c1 100644 --- a/packages/aws/data_stream/cloudwatch_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/cloudwatch_logs/elasticsearch/ingest_pipeline/default.yml @@ -1,38 +1,21 @@ --- -description: "Pipeline for CloudWatch logs" +description: "Pipeline for logs ingested from CloudWatch" processors: - - set: - field: event.ingested - value: '{{_ingest.timestamp}}' - set: field: ecs.version - value: '8.0.0' - - rename: - field: message - target_field: event.original - ignore_missing: true - - grok: - field: event.original - patterns: - - '%{TIMESTAMP_ISO8601:_tmp.timestamp} %{SYSLOGTIMESTAMP:_tmp.syslog_timestamp} %{GREEDYDATA:aws.cloudwatch.message}' - - '%{TIMESTAMP_ISO8601:_tmp.timestamp} %{GREEDYDATA:aws.cloudwatch.message}' - - date: - field: _tmp.timestamp - target_field: "@timestamp" - ignore_failure: true - formats: - - ISO8601 - - remove: - field: - - _tmp - ignore_missing: true - - remove: + value: '1.12.0' + - set: field: event.original - if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true + copy_from: message + override: false + - set: + field: cloud.provider + value: aws + - set: + field: event.kind + value: event on_failure: - set: - field: error.message + field: 'error.message' value: '{{ _ingest.on_failure_message }}' diff --git a/packages/aws/data_stream/cloudwatch_logs/fields/ecs.yml b/packages/aws/data_stream/cloudwatch_logs/fields/ecs.yml index def0bf767f1..935b468d4d0 100644 --- a/packages/aws/data_stream/cloudwatch_logs/fields/ecs.yml +++ b/packages/aws/data_stream/cloudwatch_logs/fields/ecs.yml @@ -2,5 +2,9 @@ name: ecs.version - external: ecs name: error.message +- name: message + external: ecs - external: ecs name: tags +- name: event.ingested + external: ecs diff --git a/packages/aws/data_stream/cloudwatch_logs/manifest.yml b/packages/aws/data_stream/cloudwatch_logs/manifest.yml index 49ff38e9972..42e6bc83341 100644 --- a/packages/aws/data_stream/cloudwatch_logs/manifest.yml +++ b/packages/aws/data_stream/cloudwatch_logs/manifest.yml @@ -175,3 +175,10 @@ streams: type: bool multi: false default: false + - name: data_stream.dataset + required: true + default: generic + show_user: true + title: Dataset name + description: > + Set the name for your dataset. Changing the dataset will send the data to a different index. You can't use `-` in the name of a dataset and only valid characters for [Elasticsearch index names](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html). diff --git a/packages/aws/docs/cloudwatch.md b/packages/aws/docs/cloudwatch.md index d6ab5bcca52..47f2e68c5aa 100644 --- a/packages/aws/docs/cloudwatch.md +++ b/packages/aws/docs/cloudwatch.md @@ -32,6 +32,7 @@ setup already. | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | error.message | Error message. | match_only_text | | event.dataset | Event dataset | constant_keyword | +| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | | event.module | Event module | constant_keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | @@ -49,6 +50,7 @@ setup already. | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | | tags | List of keywords used to tag each event. | keyword | diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml index cd59d869858..9952d46bcdb 100644 --- a/packages/aws/manifest.yml +++ b/packages/aws/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: aws title: AWS -version: 1.11.0 +version: 1.11.1 license: basic description: Collect logs and metrics from Amazon Web Services with Elastic Agent. type: integration From 6d1cbb25377f759324a1068138931a1e658b633a Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 24 Jan 2022 11:13:46 -0700 Subject: [PATCH 2/4] add changelog --- packages/aws/changelog.yml | 5 +++++ .../pipeline/test-cloudwatch-ec2.log-expected.json | 12 ++++++------ .../elasticsearch/ingest_pipeline/default.yml | 2 +- .../aws/data_stream/cloudwatch_logs/manifest.yml | 1 + 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml index 82b574c077f..5bc75ec1f8c 100644 --- a/packages/aws/changelog.yml +++ b/packages/aws/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.11.1" + changes: + - description: Add data_stream.dataset option for custom aws-cloudwatch log input + type: bugfix + link: https://github.com/elastic/integrations/pull/2560 - version: "1.11.0" changes: - description: Update to ECS 8.0 diff --git a/packages/aws/data_stream/cloudwatch_logs/_dev/test/pipeline/test-cloudwatch-ec2.log-expected.json b/packages/aws/data_stream/cloudwatch_logs/_dev/test/pipeline/test-cloudwatch-ec2.log-expected.json index c544ea851d1..3c70a0d3d02 100644 --- a/packages/aws/data_stream/cloudwatch_logs/_dev/test/pipeline/test-cloudwatch-ec2.log-expected.json +++ b/packages/aws/data_stream/cloudwatch_logs/_dev/test/pipeline/test-cloudwatch-ec2.log-expected.json @@ -5,7 +5,7 @@ "provider": "aws" }, "ecs": { - "version": "1.12.0" + "version": "8.0.0" }, "message": "2020-02-20T07:01:01.000Z Feb 20 07:01:01 ip-172-31-81-156 systemd: Stopping User Slice of root.", "event": { @@ -21,7 +21,7 @@ "provider": "aws" }, "ecs": { - "version": "1.12.0" + "version": "8.0.0" }, "message": "2020-02-20T07:02:18.000Z Feb 20 07:02:18 ip-172-31-81-156 dhclient[3000]: XMT: Solicit on eth0, interval 125240ms.", "event": { @@ -37,7 +37,7 @@ "provider": "aws" }, "ecs": { - "version": "1.12.0" + "version": "8.0.0" }, "message": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: DHCPREQUEST on eth0 to 172.31.80.1 port 67 (xid=0x4575af22)", "event": { @@ -53,7 +53,7 @@ "provider": "aws" }, "ecs": { - "version": "1.12.0" + "version": "8.0.0" }, "message": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: DHCPACK from 172.31.80.1 (xid=0x4575af22)", "event": { @@ -69,7 +69,7 @@ "provider": "aws" }, "ecs": { - "version": "1.12.0" + "version": "8.0.0" }, "message": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: bound to 172.31.81.156 -- renewal in 1599 seconds.", "event": { @@ -85,7 +85,7 @@ "provider": "aws" }, "ecs": { - "version": "1.12.0" + "version": "8.0.0" }, "message": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 ec2net: [get_meta] Trying to get http://169.254.169.254/latest/meta-data/network/interfaces/macs/12:e2:a9:95:8b:97/local-ipv4s", "event": { diff --git a/packages/aws/data_stream/cloudwatch_logs/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/cloudwatch_logs/elasticsearch/ingest_pipeline/default.yml index f61c3d671c1..a76c2c8d9a1 100644 --- a/packages/aws/data_stream/cloudwatch_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/cloudwatch_logs/elasticsearch/ingest_pipeline/default.yml @@ -4,7 +4,7 @@ description: "Pipeline for logs ingested from CloudWatch" processors: - set: field: ecs.version - value: '1.12.0' + value: '8.0.0' - set: field: event.original copy_from: message diff --git a/packages/aws/data_stream/cloudwatch_logs/manifest.yml b/packages/aws/data_stream/cloudwatch_logs/manifest.yml index 42e6bc83341..7eeba4d65b8 100644 --- a/packages/aws/data_stream/cloudwatch_logs/manifest.yml +++ b/packages/aws/data_stream/cloudwatch_logs/manifest.yml @@ -182,3 +182,4 @@ streams: title: Dataset name description: > Set the name for your dataset. Changing the dataset will send the data to a different index. You can't use `-` in the name of a dataset and only valid characters for [Elasticsearch index names](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html). + From 9a907d042747385ce2615cb59f9315b9f4ce1ae9 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 24 Jan 2022 12:53:36 -0700 Subject: [PATCH 3/4] change dataset option to be under advanced options --- packages/aws/data_stream/cloudwatch_logs/manifest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/aws/data_stream/cloudwatch_logs/manifest.yml b/packages/aws/data_stream/cloudwatch_logs/manifest.yml index 7eeba4d65b8..a9026017731 100644 --- a/packages/aws/data_stream/cloudwatch_logs/manifest.yml +++ b/packages/aws/data_stream/cloudwatch_logs/manifest.yml @@ -178,7 +178,7 @@ streams: - name: data_stream.dataset required: true default: generic - show_user: true + show_user: false title: Dataset name description: > Set the name for your dataset. Changing the dataset will send the data to a different index. You can't use `-` in the name of a dataset and only valid characters for [Elasticsearch index names](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html). From 34b921b884f586f21e819836ac944ebfb123a1b4 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 24 Jan 2022 13:15:39 -0700 Subject: [PATCH 4/4] add type for data_stream.dataset --- .../cloudwatch_logs/elasticsearch/ingest_pipeline/default.yml | 2 +- packages/aws/data_stream/cloudwatch_logs/manifest.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/aws/data_stream/cloudwatch_logs/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/cloudwatch_logs/elasticsearch/ingest_pipeline/default.yml index a76c2c8d9a1..af22902c87e 100644 --- a/packages/aws/data_stream/cloudwatch_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/cloudwatch_logs/elasticsearch/ingest_pipeline/default.yml @@ -17,5 +17,5 @@ processors: value: event on_failure: - set: - field: 'error.message' + field: error.message value: '{{ _ingest.on_failure_message }}' diff --git a/packages/aws/data_stream/cloudwatch_logs/manifest.yml b/packages/aws/data_stream/cloudwatch_logs/manifest.yml index a9026017731..7b1b332f4d5 100644 --- a/packages/aws/data_stream/cloudwatch_logs/manifest.yml +++ b/packages/aws/data_stream/cloudwatch_logs/manifest.yml @@ -176,6 +176,7 @@ streams: multi: false default: false - name: data_stream.dataset + type: text required: true default: generic show_user: false