[AWS] Remove duplicated number_of_workers settings from the custom …

…logs integration (#7319)

* Remove the accidental extra number_of_workers

I added an extra `number_of_workers` advanced configuration setting by
mistake while adding it to a group of CloudWatch-based integrations
missing it.

This change removes the extra definition.

I used the later setting description because it contains more details.

* Group S3 options by source

I am trying to group the S3 options by source:

- sqs queue
- bucket
  - aws bucket arn
  - non-aws bucket name

With this approach, we can define shared options like
`number_of_workers` only once. This should streamline the options and
avoid duplicated definitions.

* Update packages/aws_logs/data_stream/generic/manifest.yml

Co-authored-by: Davide Girardi <[email protected]>

* Small updates to template comments

I am trying to make template more readable.

---------

Co-authored-by: Davide Girardi <[email protected]>

packages/aws_logs/data_stream/generic/agent/stream/aws-cloudwatch.yml.hbs

@@ -62,9 +62,6 @@ api_sleep: {{ api_sleep }}
 {{#if latency }}
 latency: {{ latency }}
 {{/if}}
-{{#if number_of_workers }}
-number_of_workers: {{ number_of_workers }}
-{{/if}}
 
 {{#if credential_profile_name}}
 credential_profile_name: {{credential_profile_name}}

packages/aws_logs/data_stream/generic/agent/stream/aws-s3.yml.hbs

@@ -4,57 +4,54 @@ data_stream:
 pipeline: {{pipeline}}
 {{/if}}
 
+{{! The aws-s3 input can be configured to read from an SQS queue or an S3 bucket. }}
+
+{{! start SQS queue }}
 {{#unless bucket_arn}}
 {{#unless non_aws_bucket_name}}
 {{#if queue_url }}
 queue_url: {{ queue_url }}
 {{/if}}
 {{/unless}}
 {{/unless}}
+{{! end SQS queue }}
 
-{{#unless queue_url}}
-{{#unless non_aws_bucket_name}}
-{{#if bucket_arn }}
-bucket_arn: {{ bucket_arn }}
-{{/if}}
+{{#unless queue_url}}{{! start S3 bucket polling }}
+
+{{!
+When using an S3 bucket, you can specify only one of the following options:
+- An AWS bucket ARN
+- A non-AWS bucket name
+}}
+
+{{! shared S3 bucket polling options }}
 {{#if number_of_workers }}
 number_of_workers: {{ number_of_workers }}
 {{/if}}
+
+{{#if bucket_list_prefix }}
+bucket_list_prefix: {{ bucket_list_prefix }}
+{{/if}}
+
 {{#if bucket_list_interval }}
 bucket_list_interval: {{ bucket_list_interval }}
 {{/if}}
-{{/unless}}
+
+{{! AWS S3 bucket ARN options }}
+{{#unless non_aws_bucket_name}}
+{{#if bucket_arn }}
+bucket_arn: {{ bucket_arn }}
+{{/if}}
 {{/unless}}
 
+{{! non-AWS S3 bucket ARN options }}
 {{#unless bucket_arn}}
-{{#unless queue_url}}
 {{#if non_aws_bucket_name }}
 non_aws_bucket_name: {{ non_aws_bucket_name }}
 {{/if}}
-{{#if number_of_workers }}
-number_of_workers: {{ number_of_workers }}
-{{/if}}
-{{#if bucket_list_interval }}
-bucket_list_interval: {{ bucket_list_interval }}
-{{/if}}
-{{/unless}}
 {{/unless}}
 
-{{#unless queue_url}}
-{{#unless non_aws_bucket_name}}
-{{#unless bucket_arn }}
-{{#if bucket_list_prefix }}
-bucket_list_prefix: {{ bucket_list_prefix }}
-{{/if}}
-{{#if number_of_workers }}
-number_of_workers: {{ number_of_workers }}
-{{/if}}
-{{#if bucket_list_interval }}
-bucket_list_interval: {{ bucket_list_interval }}
-{{/if}}
-{{/unless}}
-{{/unless}}
-{{/unless}}
+{{/unless}}{{! end S3 bucket polling }}
 
 {{#if buffer_size }}
 buffer_size: {{ buffer_size }}

packages/aws_logs/data_stream/generic/manifest.yml

@@ -41,7 +41,7 @@ streams:
         default: 1
         required: false
         show_user: false
-        description: Number of workers that will process the log groups with the given log_group_name_prefix. Default value is 1.
+        description: The number of workers assigned to read from log groups. Each worker will read log events from one of the log groups matching `log_group_name_prefix`. For example, if `log_group_name_prefix` matches five log groups, then `number_of_workers` should be set to `5`. The default value is `1`.
       - name: log_streams
         type: text
         title: Log Streams
@@ -95,12 +95,6 @@ streams:
         required: false
         show_user: false
         description: "The amount of time required for the logs to be available to CloudWatch Logs. Sample values, `1m` or `5m` — see Golang [time.ParseDuration](https://pkg.go.dev/time#ParseDuration) for more details. Latency translates the query's time range to consider the CloudWatch Logs latency. Example: `5m` means that the integration will query CloudWatch to search for logs available 5 minutes ago."
-      - name: number_of_workers
-        type: integer
-        title: Number of workers
-        required: false
-        show_user: false
-        description: The number of workers assigned to reading from log groups. Each worker will read log events from one of the log groups matching `log_group_name_prefix`. For example, if `log_group_name_prefix` matches five log groups, then `number_of_workers` should be set to `5`. The default value is `1`.
       - name: tags
         type: text
         title: Tags