From 9875be855e3c6ec7c1860c04d0d5999034529656 Mon Sep 17 00:00:00 2001 From: Maurizio Branca Date: Tue, 22 Aug 2023 00:34:30 +0200 Subject: [PATCH] [AWS] Remove duplicated `number_of_workers` settings from the custom logs integration (#7319) * Remove the accidental extra number_of_workers I added an extra `number_of_workers` advanced configuration setting by mistake while adding it to a group of CloudWatch-based integrations missing it. This change removes the extra definition. I used the later setting description because it contains more details. * Group S3 options by source I am trying to group the S3 options by source: - sqs queue - bucket - aws bucket arn - non-aws bucket name With this approach, we can define shared options like `number_of_workers` only once. This should streamline the options and avoid duplicated definitions. * Update packages/aws_logs/data_stream/generic/manifest.yml Co-authored-by: Davide Girardi <1390902+girodav@users.noreply.github.com> * Small updates to template comments I am trying to make template more readable. --------- Co-authored-by: Davide Girardi <1390902+girodav@users.noreply.github.com> --- .../agent/stream/aws-cloudwatch.yml.hbs | 3 - .../generic/agent/stream/aws-s3.yml.hbs | 55 +++++++++---------- .../aws_logs/data_stream/generic/manifest.yml | 8 +-- 3 files changed, 27 insertions(+), 39 deletions(-) diff --git a/packages/aws_logs/data_stream/generic/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws_logs/data_stream/generic/agent/stream/aws-cloudwatch.yml.hbs index c141d81c552..db2cf4f64ae 100644 --- a/packages/aws_logs/data_stream/generic/agent/stream/aws-cloudwatch.yml.hbs +++ b/packages/aws_logs/data_stream/generic/agent/stream/aws-cloudwatch.yml.hbs @@ -62,9 +62,6 @@ api_sleep: {{ api_sleep }} {{#if latency }} latency: {{ latency }} {{/if}} -{{#if number_of_workers }} -number_of_workers: {{ number_of_workers }} -{{/if}} {{#if credential_profile_name}} credential_profile_name: {{credential_profile_name}} diff --git a/packages/aws_logs/data_stream/generic/agent/stream/aws-s3.yml.hbs b/packages/aws_logs/data_stream/generic/agent/stream/aws-s3.yml.hbs index 74f5880d9cc..d43668bf1b2 100644 --- a/packages/aws_logs/data_stream/generic/agent/stream/aws-s3.yml.hbs +++ b/packages/aws_logs/data_stream/generic/agent/stream/aws-s3.yml.hbs @@ -4,6 +4,9 @@ data_stream: pipeline: {{pipeline}} {{/if}} +{{! The aws-s3 input can be configured to read from an SQS queue or an S3 bucket. }} + +{{! start SQS queue }} {{#unless bucket_arn}} {{#unless non_aws_bucket_name}} {{#if queue_url }} @@ -11,50 +14,44 @@ queue_url: {{ queue_url }} {{/if}} {{/unless}} {{/unless}} +{{! end SQS queue }} -{{#unless queue_url}} -{{#unless non_aws_bucket_name}} -{{#if bucket_arn }} -bucket_arn: {{ bucket_arn }} -{{/if}} +{{#unless queue_url}}{{! start S3 bucket polling }} + +{{! +When using an S3 bucket, you can specify only one of the following options: +- An AWS bucket ARN +- A non-AWS bucket name +}} + +{{! shared S3 bucket polling options }} {{#if number_of_workers }} number_of_workers: {{ number_of_workers }} {{/if}} + +{{#if bucket_list_prefix }} +bucket_list_prefix: {{ bucket_list_prefix }} +{{/if}} + {{#if bucket_list_interval }} bucket_list_interval: {{ bucket_list_interval }} {{/if}} -{{/unless}} + +{{! AWS S3 bucket ARN options }} +{{#unless non_aws_bucket_name}} +{{#if bucket_arn }} +bucket_arn: {{ bucket_arn }} +{{/if}} {{/unless}} +{{! non-AWS S3 bucket ARN options }} {{#unless bucket_arn}} -{{#unless queue_url}} {{#if non_aws_bucket_name }} non_aws_bucket_name: {{ non_aws_bucket_name }} {{/if}} -{{#if number_of_workers }} -number_of_workers: {{ number_of_workers }} -{{/if}} -{{#if bucket_list_interval }} -bucket_list_interval: {{ bucket_list_interval }} -{{/if}} -{{/unless}} {{/unless}} -{{#unless queue_url}} -{{#unless non_aws_bucket_name}} -{{#unless bucket_arn }} -{{#if bucket_list_prefix }} -bucket_list_prefix: {{ bucket_list_prefix }} -{{/if}} -{{#if number_of_workers }} -number_of_workers: {{ number_of_workers }} -{{/if}} -{{#if bucket_list_interval }} -bucket_list_interval: {{ bucket_list_interval }} -{{/if}} -{{/unless}} -{{/unless}} -{{/unless}} +{{/unless}}{{! end S3 bucket polling }} {{#if buffer_size }} buffer_size: {{ buffer_size }} diff --git a/packages/aws_logs/data_stream/generic/manifest.yml b/packages/aws_logs/data_stream/generic/manifest.yml index cd3826ec1f5..538fa581622 100644 --- a/packages/aws_logs/data_stream/generic/manifest.yml +++ b/packages/aws_logs/data_stream/generic/manifest.yml @@ -41,7 +41,7 @@ streams: default: 1 required: false show_user: false - description: Number of workers that will process the log groups with the given log_group_name_prefix. Default value is 1. + description: The number of workers assigned to read from log groups. Each worker will read log events from one of the log groups matching `log_group_name_prefix`. For example, if `log_group_name_prefix` matches five log groups, then `number_of_workers` should be set to `5`. The default value is `1`. - name: log_streams type: text title: Log Streams @@ -95,12 +95,6 @@ streams: required: false show_user: false description: "The amount of time required for the logs to be available to CloudWatch Logs. Sample values, `1m` or `5m` — see Golang [time.ParseDuration](https://pkg.go.dev/time#ParseDuration) for more details. Latency translates the query's time range to consider the CloudWatch Logs latency. Example: `5m` means that the integration will query CloudWatch to search for logs available 5 minutes ago." - - name: number_of_workers - type: integer - title: Number of workers - required: false - show_user: false - description: The number of workers assigned to reading from log groups. Each worker will read log events from one of the log groups matching `log_group_name_prefix`. For example, if `log_group_name_prefix` matches five log groups, then `number_of_workers` should be set to `5`. The default value is `1`. - name: tags type: text title: Tags