diff --git a/.github/workflows/bump-elastic-stack-version.yml b/.github/workflows/bump-elastic-stack-version.yml index 0ef4f5b97..0d8f93748 100644 --- a/.github/workflows/bump-elastic-stack-version.yml +++ b/.github/workflows/bump-elastic-stack-version.yml @@ -35,7 +35,7 @@ jobs: echo "UPDATECLI_ACTION=apply" >> $GITHUB_ENV - name: Install Updatecli in the runner - uses: updatecli/updatecli-action@d0950ebbe80f4f80c3392b288d6a218fae872f69 #v2.66.0 + uses: updatecli/updatecli-action@6b8881a17fc8038e884ec94ff72a49e8e8a4069f #v2.67.0 - name: Update default stack version # --experimental needed for commitusingapi option. diff --git a/Makefile b/Makefile index 2d74b317d..00eba52ad 100644 --- a/Makefile +++ b/Makefile @@ -73,7 +73,7 @@ test-stack-command-86: ./scripts/test-stack-command.sh 8.6.2 test-stack-command-8x: - ./scripts/test-stack-command.sh 8.16.0-77b64b1f-SNAPSHOT + ./scripts/test-stack-command.sh 8.16.0-a3caaa65-SNAPSHOT test-stack-command-with-apm-server: APM_SERVER_ENABLED=true ./scripts/test-stack-command.sh diff --git a/internal/install/application_configuration.go b/internal/install/application_configuration.go index f2e76434e..8f8366e0c 100644 --- a/internal/install/application_configuration.go +++ b/internal/install/application_configuration.go @@ -162,10 +162,10 @@ func selectElasticAgentImageName(version, agentBaseImage string) string { return elasticAgentLegacyImageName } - disableWolfiImages := false + disableWolfiImages := true valueEnv, ok := os.LookupEnv(disableElasticAgentWolfiEnvVar) - if ok && strings.ToLower(valueEnv) != "false" { - disableWolfiImages = true + if ok && strings.ToLower(valueEnv) != "true" { + disableWolfiImages = false } switch { case agentBaseImage == "complete": diff --git a/internal/install/application_configuration_test.go b/internal/install/application_configuration_test.go index d5c4b6507..3350a8005 100644 --- a/internal/install/application_configuration_test.go +++ b/internal/install/application_configuration_test.go @@ -52,10 +52,10 @@ func TestSelectElasticAgentImageName_NextStackInOwnNamespace(t *testing.T) { assert.Equal(t, selected, elasticAgentCompleteImageName) } -func TestSelectElasticAgentImageName_WolfiImage(t *testing.T) { +func TestSelectElasticAgentImageName_DefaultImage816(t *testing.T) { version := stackVersion8160 selected := selectElasticAgentImageName(version, "") - assert.Equal(t, selected, elasticAgentWolfiImageName) + assert.Equal(t, selected, elasticAgentCompleteImageName) } func TestSelectElasticAgentImageName_DisableWolfiImageEnvVar(t *testing.T) { @@ -79,7 +79,7 @@ func TestSelectCompleteElasticAgentImageName_ForceCompleteImage(t *testing.T) { func TestSelectCompleteElasticAgentImageName_ForceDefaultImage(t *testing.T) { version := stackVersion8160 selected := selectElasticAgentImageName(version, "default") - assert.Equal(t, selected, elasticAgentWolfiImageName) + assert.Equal(t, selected, elasticAgentCompleteImageName) } func TestSelectCompleteElasticAgentImageName_ForceDefaultImageOldStack(t *testing.T) { diff --git a/internal/servicedeployer/_static/terraform_deployer.yml b/internal/servicedeployer/_static/terraform_deployer.yml index 5e7173e32..f775bb93e 100644 --- a/internal/servicedeployer/_static/terraform_deployer.yml +++ b/internal/servicedeployer/_static/terraform_deployer.yml @@ -13,3 +13,4 @@ services: volumes: - ${TF_DIR}:/stage - ${TF_OUTPUT_DIR}:/output + - ${SERVICE_LOGS_DIR}:/tmp/service_logs/ diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 31e515c0a..d210f2d09 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -378,7 +378,6 @@ func (r *runner) getAllVariants(folder testrunner.TestFolder) ([]string, error) if len(variants) == 1 && variants[0] == "" { logger.Debug("No variant mode") } - logger.Debugf(">>>>>> number of variants loaded: %d - %q", len(variants), strings.Join(variants, ",")) } return variants, nil diff --git a/test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/env.yml b/test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/env.yml deleted file mode 100644 index b795fcdeb..000000000 --- a/test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/env.yml +++ /dev/null @@ -1,9 +0,0 @@ -version: '2.3' -services: - terraform: - environment: - - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} - - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} - - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} - - AWS_PROFILE=${AWS_PROFILE} - - AWS_REGION=${AWS_REGION:-us-east-1} diff --git a/test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/main.tf b/test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/main.tf deleted file mode 100644 index fff56b462..000000000 --- a/test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/main.tf +++ /dev/null @@ -1,42 +0,0 @@ -variable "TEST_RUN_ID" { - default = "detached" -} - -provider "aws" { - default_tags { - tags = { - environment = var.ENVIRONMENT - repo = var.REPO - branch = var.BRANCH - build = var.BUILD_ID - created_date = var.CREATED_DATE - - division = "engineering" - org = "ingest" - team = "ecosystem" - project = "ci-testing" - } - } -} - -resource "aws_instance" "i" { - ami = data.aws_ami.latest-amzn.id - monitoring = true - instance_type = "t1.micro" - tags = { - Name = "elastic-package-test-${var.TEST_RUN_ID}" - } -} - -data "aws_ami" "latest-amzn" { - most_recent = true - owners = [ "amazon" ] # AWS - filter { - name = "name" - values = ["amzn2-ami-minimal-hvm-*-ebs"] - } -} - -output "instance_id" { - value = aws_instance.i.id -} \ No newline at end of file diff --git a/test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/variables.tf b/test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/variables.tf deleted file mode 100644 index 9d78b1b3c..000000000 --- a/test/packages/parallel/aws/data_stream/ec2_metrics/_dev/deploy/tf/variables.tf +++ /dev/null @@ -1,22 +0,0 @@ -variable "BRANCH" { - description = "Branch name or pull request for tagging purposes" - default = "unknown-branch" -} - -variable "BUILD_ID" { - description = "Build ID in the CI for tagging purposes" - default = "unknown-build" -} - -variable "CREATED_DATE" { - description = "Creation date in epoch time for tagging purposes" - default = "unknown-date" -} - -variable "ENVIRONMENT" { - default = "unknown-environment" -} - -variable "REPO" { - default = "unknown-repo-name" -} diff --git a/test/packages/parallel/aws/data_stream/ec2_metrics/_dev/test/system/test-default-config.yml b/test/packages/parallel/aws/data_stream/ec2_metrics/_dev/test/system/test-default-config.yml deleted file mode 100644 index edbd95621..000000000 --- a/test/packages/parallel/aws/data_stream/ec2_metrics/_dev/test/system/test-default-config.yml +++ /dev/null @@ -1,13 +0,0 @@ -wait_for_data_timeout: 20m # AWS CloudWatch may delay metrics delivery for more than 10 minutes. -vars: - access_key_id: '{{AWS_ACCESS_KEY_ID}}' - secret_access_key: '{{AWS_SECRET_ACCESS_KEY}}' - session_token: '{{AWS_SESSION_TOKEN}}' -data_stream: - vars: - period: 5m - latency: 10m - instance_id: '{{TF_OUTPUT_instance_id}}' - tags_filter: |- - - key: Name - value: "elastic-package-test-{{TEST_RUN_ID}}" diff --git a/test/packages/parallel/aws_logs/_dev/build/build.yml b/test/packages/parallel/aws_logs/_dev/build/build.yml deleted file mode 100644 index 47cbed9fe..000000000 --- a/test/packages/parallel/aws_logs/_dev/build/build.yml +++ /dev/null @@ -1,3 +0,0 @@ -dependencies: - ecs: - reference: git@v8.0.0 diff --git a/test/packages/parallel/aws_logs/_dev/build/docs/README.md b/test/packages/parallel/aws_logs/_dev/build/docs/README.md deleted file mode 100644 index 1b16353c9..000000000 --- a/test/packages/parallel/aws_logs/_dev/build/docs/README.md +++ /dev/null @@ -1,28 +0,0 @@ -# Custom AWS Log Integration - -The custom AWS input integration offers users two ways to collect logs from AWS: from an [S3 bucket](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ServerLogs.html) (with or without SQS notification) and from [CloudWatch](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/WhatIsCloudWatchLogs.html). -Custom ingest pipelines may be added by adding the name to the pipeline configuration option, creating custom ingest pipelines can be done either through the API or the [Ingest Node Pipeline UI](/app/management/ingest/ingest_pipelines/). - -## Collecting logs from S3 bucket - -When collecting logs from S3 bucket is enabled, users can retrieve logs from S3 -objects that are pointed to by S3 notification events read from an SQS queue or -directly polling list of S3 objects in an S3 bucket. - -The use of SQS notification is preferred: polling list of S3 objects is -expensive in terms of performance and costs and should be preferably used only -when no SQS notification can be attached to the S3 buckets. This input -integration also supports S3 notification from SNS to SQS. - -SQS notification method is enabled setting `queue_url` configuration value. S3 -bucket list polling method is enabled setting `bucket_arn` configuration value -and `number_of_workers` value. Both `queue_url` and `bucket_arn` cannot be set -at the same time and at least one of the two value must be set. - -## Collecting logs from CloudWatch - -When collecting logs from CloudWatch is enabled, users can retrieve logs from -all log streams in a specific log group. `filterLogEvents` AWS API is used to -list log events from the specified log group. Amazon CloudWatch Logs can be used -to store log files from Amazon Elastic Compute Cloud(EC2), AWS CloudTrail, -Route53, and other sources. diff --git a/test/packages/parallel/aws_logs/changelog.yml b/test/packages/parallel/aws_logs/changelog.yml deleted file mode 100644 index c1469c65b..000000000 --- a/test/packages/parallel/aws_logs/changelog.yml +++ /dev/null @@ -1,65 +0,0 @@ -- version: "999.999.999" - changes: - - description: Change test public IPs to the supported subset - type: bugfix - link: https://github.com/elastic/integrations/pull/2327 -- version: "0.4.0" - changes: - - description: Add multiline support for using s3 input - type: enhancement - link: https://github.com/elastic/integrations/pull/6081 -- version: "0.3.3" - changes: - - description: Added categories and/or subcategories. - type: enhancement - link: https://github.com/elastic/integrations/pull/5123 -- version: "0.3.2" - changes: - - description: Add required field number of workers to support non aws buckets, and add default value. - type: enhancement - link: https://github.com/elastic/integrations/pull/4917 -- version: "0.3.1" - changes: - - description: Add latency config parameter for aws-cloudwatch input - type: bugfix - link: https://github.com/elastic/integrations/pull/4859 -- version: "0.3.0" - changes: - - description: Expose Default Region setting to UI - type: enhancement - link: https://github.com/elastic/integrations/pull/4158 -- version: "0.2.5" - changes: - - description: Set default endpoint to empty string - type: bugfix - link: https://github.com/elastic/integrations/pull/4103 -- version: "0.2.4" - changes: - - description: Fix proxy URL documentation rendering. - type: bugfix - link: https://github.com/elastic/integrations/pull/3881 -- version: "0.2.3" - changes: - - description: Fix misspelling of Log Stream Prefix variable in manifest for aws-cloudwatch input - type: bugfix - link: https://github.com/elastic/integrations/pull/3610 -- version: "0.2.2" - changes: - - description: update readme file - type: bugfix - link: https://github.com/elastic/integrations/pull/2819 -- version: "0.2.1" - changes: - - description: Add kibana version constraint - type: bugfix - link: https://github.com/elastic/integrations/pull/2819 -- version: "0.2.0" - changes: - - description: Move s3 input and cloudwatch input into the same generic data stream - type: enhancement - link: https://github.com/elastic/integrations/pull/2710 -- version: "0.1.0" - changes: - - description: initial release - type: enhancement # can be one of: enhancement, bugfix, breaking-change - link: https://github.com/elastic/integrations/pull/2353 diff --git a/test/packages/parallel/aws_logs/data_stream/generic/_dev/deploy/tf/env.yml b/test/packages/parallel/aws_logs/data_stream/generic/_dev/deploy/tf/env.yml deleted file mode 100644 index b795fcdeb..000000000 --- a/test/packages/parallel/aws_logs/data_stream/generic/_dev/deploy/tf/env.yml +++ /dev/null @@ -1,9 +0,0 @@ -version: '2.3' -services: - terraform: - environment: - - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} - - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} - - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} - - AWS_PROFILE=${AWS_PROFILE} - - AWS_REGION=${AWS_REGION:-us-east-1} diff --git a/test/packages/parallel/aws_logs/data_stream/generic/_dev/deploy/tf/main.tf b/test/packages/parallel/aws_logs/data_stream/generic/_dev/deploy/tf/main.tf deleted file mode 100644 index f7d5618cd..000000000 --- a/test/packages/parallel/aws_logs/data_stream/generic/_dev/deploy/tf/main.tf +++ /dev/null @@ -1,64 +0,0 @@ -variable "TEST_RUN_ID" { - default = "detached" -} - -provider "aws" { - default_tags { - tags = { - environment = var.ENVIRONMENT - repo = var.REPO - branch = var.BRANCH - build = var.BUILD_ID - created_date = var.CREATED_DATE - } - } -} - -resource "aws_s3_bucket" "bucket" { - bucket = "elastic-package-aws-logs-bucket-${var.TEST_RUN_ID}" -} - -resource "aws_sqs_queue" "queue" { - name = "elastic-package-aws-logs-queue-${var.TEST_RUN_ID}" - policy = < - If the host is a container. - - - name: os.build - type: keyword - example: "18D109" - description: > - OS build information. - - - name: os.codename - type: keyword - example: "stretch" - description: > - OS codename, if any. - diff --git a/test/packages/parallel/aws_logs/data_stream/generic/fields/ecs.yml b/test/packages/parallel/aws_logs/data_stream/generic/fields/ecs.yml deleted file mode 100644 index d1c52ea0e..000000000 --- a/test/packages/parallel/aws_logs/data_stream/generic/fields/ecs.yml +++ /dev/null @@ -1,10 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: log.level -- external: ecs - name: message -- name: event.original - external: ecs -- name: tags - external: ecs diff --git a/test/packages/parallel/aws_logs/data_stream/generic/fields/fields.yml b/test/packages/parallel/aws_logs/data_stream/generic/fields/fields.yml deleted file mode 100644 index c43e4dcc5..000000000 --- a/test/packages/parallel/aws_logs/data_stream/generic/fields/fields.yml +++ /dev/null @@ -1,28 +0,0 @@ -- name: aws.cloudwatch - type: group - fields: - - name: message - type: text - description: | - CloudWatch log message. -- name: aws.s3 - type: group - fields: - - name: bucket - type: group - fields: - - name: name - type: keyword - description: Name of the S3 bucket that this log retrieved from. - - name: arn - type: keyword - description: ARN of the S3 bucket that this log retrieved from. - - name: object.key - type: keyword - description: Name of the S3 object that this log retrieved from. -- name: log.file.path - type: keyword - description: Path to the log file. -- name: log.offset - type: long - description: Log offset diff --git a/test/packages/parallel/aws_logs/data_stream/generic/manifest.yml b/test/packages/parallel/aws_logs/data_stream/generic/manifest.yml deleted file mode 100644 index 7710d3c50..000000000 --- a/test/packages/parallel/aws_logs/data_stream/generic/manifest.yml +++ /dev/null @@ -1,228 +0,0 @@ -title: Custom logs from AWS -type: logs -streams: - - input: aws-s3 - enabled: false - template_path: aws-s3.yml.hbs - title: AWS logs from S3 - description: Collect logs using aws-s3 input with or without SQS notification - vars: - - name: api_timeout - type: text - title: API Timeout - multi: false - required: false - show_user: false - description: The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. - - name: bucket_arn - type: text - title: Bucket ARN - multi: false - required: false - show_user: true - description: ARN of the AWS S3 bucket that will be polled for list operation. (Required when `queue_url` and `non_aws_bucket_name` are not set). - - name: number_of_workers - type: integer - title: Number of Workers - multi: false - required: false - default: 1 - show_user: true - description: Number of workers that will process the S3 objects listed. (Required when `bucket_arn` is set). - - name: bucket_list_interval - type: text - title: Bucket List Interval - multi: false - required: false - show_user: false - default: 120s - description: Time interval for polling listing of the S3 bucket. - - name: bucket_list_prefix - type: text - title: Bucket List Prefix - multi: false - required: false - show_user: false - description: Prefix to apply for the list request to the S3 bucket. - - name: buffer_size - type: text - title: Buffer Size - multi: false - required: false - show_user: false - description: The size in bytes of the buffer that each harvester uses when fetching a file. This only applies to non-JSON logs. - - name: content_type - type: text - title: Content Type - multi: false - required: false - show_user: false - description: > - A standard MIME type describing the format of the object data. This can be set to override the MIME type that was given to the object when it was uploaded. For example application/json. - - - name: encoding - type: text - title: Encoding - multi: false - required: false - show_user: false - description: The file encoding to use for reading data that contains international characters. This only applies to non-JSON logs. - - name: expand_event_list_from_field - type: text - title: Expand Event List from Field - multi: false - required: false - show_user: false - description: > - If the fileset using this input expects to receive multiple messages bundled under a specific field then the config option expand_event_list_from_field value can be assigned the name of the field. This setting will be able to split the messages under the group value into separate events. For example, CloudTrail logs are in JSON format and events are found under the JSON object "Records". - - - name: file_selectors - type: yaml - title: File Selectors - multi: true - required: false - show_user: false - description: > - If the SQS queue will have events that correspond to files that this integration shouldn’t process file_selectors can be used to limit the files that are downloaded. This is a list of selectors which are made up of regex and expand_event_list_from_field options. The regex should match the S3 object key in the SQS message, and the optional expand_event_list_from_field is the same as the global setting. If file_selectors is given, then any global expand_event_list_from_field value is ignored in favor of the ones specified in the file_selectors. Regex syntax is the same as the Go language. Files that don’t match one of the regexes won’t be processed. content_type, parsers, include_s3_metadata,max_bytes, buffer_size, and encoding may also be set for each file selector. - - - name: fips_enabled - type: bool - title: Enable S3 FIPS - default: false - multi: false - required: false - show_user: false - description: Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. - - name: include_s3_metadata - type: text - title: Include S3 Metadata - multi: true - required: false - show_user: false - description: > - This input can include S3 object metadata in the generated events for use in follow-on processing. You must specify the list of keys to include. By default none are included. If the key exists in the S3 response then it will be included in the event as aws.s3.metadata. where the key name as been normalized to all lowercase. - - - name: max_bytes - type: text - title: Max Bytes - default: 10MiB - multi: false - required: false - show_user: false - description: The maximum number of bytes that a single log message can have. All bytes after max_bytes are discarded and not sent. This setting is especially useful for multiline log messages, which can get large. This only applies to non-JSON logs. - - name: max_number_of_messages - type: integer - title: Maximum Concurrent SQS Messages - description: The maximum number of SQS messages that can be inflight at any time. - default: 5 - required: false - show_user: false - - name: parsers - type: yaml - title: Parsers - description: | - This option expects a list of parsers that the payload has to go through. For more information see [Parsers](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html#input-aws-s3-parsers) - required: false - show_user: true - multi: false - default: | - #- multiline: - # pattern: "^ - Enabling this option sets the bucket name as a path in the API call instead of a subdomain. When enabled https://.s3...com becomes https://s3...com/. This is only supported with 3rd party S3 providers. AWS does not support path style. - - - name: provider - type: text - title: Provider Name - multi: false - required: false - show_user: false - description: Name of the 3rd party S3 bucket provider like backblaze or GCP. - - name: queue_url - type: text - title: Queue URL - multi: false - required: false - show_user: true - description: URL of the AWS SQS queue that messages will be received from. - - name: sqs.max_receive_count - type: integer - title: SQS Message Maximum Receive Count - multi: false - required: false - show_user: false - default: 5 - description: The maximum number of times a SQS message should be received (retried) before deleting it. This feature prevents poison-pill messages (messages that can be received but can’t be processed) from consuming resources. - - name: sqs.wait_time - type: text - title: SQS Maximum Wait Time - multi: false - required: false - show_user: false - default: 20s - description: > - The maximum duration that an SQS `ReceiveMessage` call should wait for a message to arrive in the queue before returning. The maximum value is `20s`. - - - name: visibility_timeout - type: text - title: Visibility Timeout - multi: false - required: false - show_user: false - description: The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours. - - name: tags - type: text - title: Tags - multi: true - required: true - show_user: true - default: - - forwarded - - name: processors - type: yaml - title: Processors - multi: false - required: false - show_user: false - description: > - Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. - - - name: preserve_original_event - required: true - show_user: true - title: Preserve original event - description: Preserves a raw copy of the original event, added to the field `event.original` - type: bool - multi: false - default: false - - name: data_stream.dataset - type: text - required: true - default: aws_logs.generic - show_user: true - title: Dataset name - description: > - Set the name for your dataset. Changing the dataset will send the data to a different index. You can't use `-` in the name of a dataset and only valid characters for [Elasticsearch index names](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html). - - - name: pipeline - type: text - title: Ingest Pipeline - description: | - The Ingest Node pipeline ID to be used by the integration. - required: false - show_user: true diff --git a/test/packages/parallel/aws_logs/data_stream/generic/sample_event.json b/test/packages/parallel/aws_logs/data_stream/generic/sample_event.json deleted file mode 100644 index e07cb4fe6..000000000 --- a/test/packages/parallel/aws_logs/data_stream/generic/sample_event.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "@timestamp": "2020-02-20T07:02:37.000Z", - "data_stream": { - "namespace": "default", - "type": "logs", - "dataset": "aws_logs.generic" - }, - "ecs": { - "version": "8.0.0" - }, - "event": { - "ingested": "2021-07-19T21:47:04.696803300Z", - "original": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 ec2net: [get_meta] Trying to get http://169.254.169.254/latest/meta-data/network/interfaces/macs/12:e2:a9:95:8b:97/local-ipv4s", - "kind": "event", - "dataset": "aws_logs.generic" - }, - "message": "ip-172-31-81-156 ec2net: [get_meta] Trying to get http://169.254.169.254/latest/meta-data/network/interfaces/macs/12:e2:a9:95:8b:97/local-ipv4s", - "tags": [ - "preserve_original_event" - ] -} \ No newline at end of file diff --git a/test/packages/parallel/aws_logs/docs/README.md b/test/packages/parallel/aws_logs/docs/README.md deleted file mode 100644 index 1b16353c9..000000000 --- a/test/packages/parallel/aws_logs/docs/README.md +++ /dev/null @@ -1,28 +0,0 @@ -# Custom AWS Log Integration - -The custom AWS input integration offers users two ways to collect logs from AWS: from an [S3 bucket](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ServerLogs.html) (with or without SQS notification) and from [CloudWatch](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/WhatIsCloudWatchLogs.html). -Custom ingest pipelines may be added by adding the name to the pipeline configuration option, creating custom ingest pipelines can be done either through the API or the [Ingest Node Pipeline UI](/app/management/ingest/ingest_pipelines/). - -## Collecting logs from S3 bucket - -When collecting logs from S3 bucket is enabled, users can retrieve logs from S3 -objects that are pointed to by S3 notification events read from an SQS queue or -directly polling list of S3 objects in an S3 bucket. - -The use of SQS notification is preferred: polling list of S3 objects is -expensive in terms of performance and costs and should be preferably used only -when no SQS notification can be attached to the S3 buckets. This input -integration also supports S3 notification from SNS to SQS. - -SQS notification method is enabled setting `queue_url` configuration value. S3 -bucket list polling method is enabled setting `bucket_arn` configuration value -and `number_of_workers` value. Both `queue_url` and `bucket_arn` cannot be set -at the same time and at least one of the two value must be set. - -## Collecting logs from CloudWatch - -When collecting logs from CloudWatch is enabled, users can retrieve logs from -all log streams in a specific log group. `filterLogEvents` AWS API is used to -list log events from the specified log group. Amazon CloudWatch Logs can be used -to store log files from Amazon Elastic Compute Cloud(EC2), AWS CloudTrail, -Route53, and other sources. diff --git a/test/packages/parallel/aws_logs/img/icon.svg b/test/packages/parallel/aws_logs/img/icon.svg deleted file mode 100644 index e60377c8b..000000000 --- a/test/packages/parallel/aws_logs/img/icon.svg +++ /dev/null @@ -1,5 +0,0 @@ - - - - - diff --git a/test/packages/parallel/aws_logs/manifest.yml b/test/packages/parallel/aws_logs/manifest.yml deleted file mode 100644 index fe7e906ab..000000000 --- a/test/packages/parallel/aws_logs/manifest.yml +++ /dev/null @@ -1,94 +0,0 @@ -format_version: 1.0.0 -name: aws_logs -title: Custom AWS Logs -description: Collect raw logs from AWS S3 or CloudWatch with Elastic Agent. -type: integration -# version is set to something very large to so this test package can -# be installed in the package registry regardless of the version of -# the actual apache package in the registry at any given time. -version: 999.999.999 -release: beta -license: basic -categories: - - observability - - custom - - aws -conditions: - kibana.version: ^8.0.0 -policy_templates: - - name: aws_logs - title: Custom AWS Logs - description: Collect raw logs from AWS S3 or CloudWatch with Elastic Agent. - inputs: - - type: aws-s3 - title: Collect Logs from S3 Bucket - description: Collect raw logs from S3 bucket with Elastic Agent. - - type: aws-cloudwatch - title: Collect Logs from CloudWatch - description: Collect raw logs from CloudWatch with Elastic Agent. -icons: - - src: "/img/icon.svg" - type: "image/svg+xml" -vars: - - name: shared_credential_file - type: text - title: Shared Credential File - multi: false - required: false - show_user: false - description: Directory of the shared credentials file - - name: credential_profile_name - type: text - title: Credential Profile Name - multi: false - required: false - show_user: false - - name: access_key_id - type: text - title: Access Key ID - multi: false - required: false - show_user: true - - name: secret_access_key - type: text - title: Secret Access Key - multi: false - required: false - show_user: true - - name: session_token - type: text - title: Session Token - multi: false - required: false - show_user: true - - name: role_arn - type: text - title: Role ARN - multi: false - required: false - show_user: false - - name: endpoint - type: text - title: Endpoint - multi: false - required: false - show_user: false - default: "" - description: URL of the entry point for an AWS web service - - name: default_region - type: text - title: Default AWS Region - multi: false - required: false - show_user: false - default: "" - description: Default region to use prior to connecting to region specific services/endpoints if no AWS region is set from environment variable, credentials or instance profile. If none of the above are set and no default region is set as well, `us-east-1` is used. A region, either from environment variable, credentials or instance profile or from this default region setting, needs to be set when using regions in non-regular AWS environments such as AWS China or US Government Isolated. - - name: proxy_url - type: text - title: Proxy URL - multi: false - required: false - show_user: false - description: URL to proxy connections in the form of http\[s\]://:@: -owner: - github: elastic/obs-cloud-monitoring diff --git a/test/packages/parallel/terraform_local/LICENSE.txt b/test/packages/parallel/terraform_local/LICENSE.txt new file mode 100644 index 000000000..809108b85 --- /dev/null +++ b/test/packages/parallel/terraform_local/LICENSE.txt @@ -0,0 +1,93 @@ +Elastic License 2.0 + +URL: https://www.elastic.co/licensing/elastic-license + +## Acceptance + +By using the software, you agree to all of the terms and conditions below. + +## Copyright License + +The licensor grants you a non-exclusive, royalty-free, worldwide, +non-sublicensable, non-transferable license to use, copy, distribute, make +available, and prepare derivative works of the software, in each case subject to +the limitations and conditions below. + +## Limitations + +You may not provide the software to third parties as a hosted or managed +service, where the service provides users with access to any substantial set of +the features or functionality of the software. + +You may not move, change, disable, or circumvent the license key functionality +in the software, and you may not remove or obscure any functionality in the +software that is protected by the license key. + +You may not alter, remove, or obscure any licensing, copyright, or other notices +of the licensor in the software. Any use of the licensor’s trademarks is subject +to applicable law. + +## Patents + +The licensor grants you a license, under any patent claims the licensor can +license, or becomes able to license, to make, have made, use, sell, offer for +sale, import and have imported the software, in each case subject to the +limitations and conditions in this license. This license does not cover any +patent claims that you cause to be infringed by modifications or additions to +the software. If you or your company make any written claim that the software +infringes or contributes to infringement of any patent, your patent license for +the software granted under these terms ends immediately. If your company makes +such a claim, your patent license ends immediately for work on behalf of your +company. + +## Notices + +You must ensure that anyone who gets a copy of any part of the software from you +also gets a copy of these terms. + +If you modify the software, you must include in any modified copies of the +software prominent notices stating that you have modified the software. + +## No Other Rights + +These terms do not imply any licenses other than those expressly granted in +these terms. + +## Termination + +If you use the software in violation of these terms, such use is not licensed, +and your licenses will automatically terminate. If the licensor provides you +with a notice of your violation, and you cease all violation of this license no +later than 30 days after you receive that notice, your licenses will be +reinstated retroactively. However, if you violate these terms after such +reinstatement, any additional violation of these terms will cause your licenses +to terminate automatically and permanently. + +## No Liability + +*As far as the law allows, the software comes as is, without any warranty or +condition, and the licensor will not be liable to you for any damages arising +out of these terms or the use or nature of the software, under any kind of +legal claim.* + +## Definitions + +The **licensor** is the entity offering these terms, and the **software** is the +software the licensor makes available under these terms, including any portion +of it. + +**you** refers to the individual or entity agreeing to these terms. + +**your company** is any legal entity, sole proprietorship, or other kind of +organization that you work for, plus all organizations that have control over, +are under the control of, or are under common control with that +organization. **control** means ownership of substantially all the assets of an +entity, or the power to direct its management and policies by vote, contract, or +otherwise. Control can be direct or indirect. + +**your licenses** are all the licenses granted to you for the software under +these terms. + +**use** means anything you do with the software requiring one of your licenses. + +**trademark** means trademarks, service marks, and similar rights. diff --git a/test/packages/parallel/terraform_local/_dev/build/build.yml b/test/packages/parallel/terraform_local/_dev/build/build.yml new file mode 100644 index 000000000..e2b012548 --- /dev/null +++ b/test/packages/parallel/terraform_local/_dev/build/build.yml @@ -0,0 +1,3 @@ +dependencies: + ecs: + reference: git@v8.11.0 diff --git a/test/packages/parallel/terraform_local/changelog.yml b/test/packages/parallel/terraform_local/changelog.yml new file mode 100644 index 000000000..bb0320a52 --- /dev/null +++ b/test/packages/parallel/terraform_local/changelog.yml @@ -0,0 +1,6 @@ +# newer versions go on top +- version: "0.0.1" + changes: + - description: Initial draft of the package + type: enhancement + link: https://github.com/elastic/integrations/pull/1 # FIXME Replace with the real PR link diff --git a/test/packages/parallel/terraform_local/data_stream/local/_dev/deploy/tf/files/example.log b/test/packages/parallel/terraform_local/data_stream/local/_dev/deploy/tf/files/example.log new file mode 100644 index 000000000..8701353d8 --- /dev/null +++ b/test/packages/parallel/terraform_local/data_stream/local/_dev/deploy/tf/files/example.log @@ -0,0 +1 @@ +{ "contents": "Message from file", "file": "logs.json"} diff --git a/test/packages/parallel/terraform_local/data_stream/local/_dev/deploy/tf/main.tf b/test/packages/parallel/terraform_local/data_stream/local/_dev/deploy/tf/main.tf new file mode 100644 index 000000000..00ded29f8 --- /dev/null +++ b/test/packages/parallel/terraform_local/data_stream/local/_dev/deploy/tf/main.tf @@ -0,0 +1,5 @@ +resource "local_file" "log" { + source = "./files/example.log" + filename = "/tmp/service_logs/file.log" + file_permission = "0777" +} diff --git a/test/packages/parallel/terraform_local/data_stream/local/_dev/test/system/test-default-config.yml b/test/packages/parallel/terraform_local/data_stream/local/_dev/test/system/test-default-config.yml new file mode 100644 index 000000000..b10b64f08 --- /dev/null +++ b/test/packages/parallel/terraform_local/data_stream/local/_dev/test/system/test-default-config.yml @@ -0,0 +1,2 @@ +wait_for_data_timeout: 1m +vars: ~ diff --git a/test/packages/parallel/terraform_local/data_stream/local/agent/stream/stream.yml.hbs b/test/packages/parallel/terraform_local/data_stream/local/agent/stream/stream.yml.hbs new file mode 100644 index 000000000..5845510de --- /dev/null +++ b/test/packages/parallel/terraform_local/data_stream/local/agent/stream/stream.yml.hbs @@ -0,0 +1,7 @@ +paths: +{{#each paths as |path i|}} + - {{path}} +{{/each}} +exclude_files: [".gz$"] +processors: + - add_locale: ~ diff --git a/test/packages/parallel/terraform_local/data_stream/local/elasticsearch/ingest_pipeline/default.yml b/test/packages/parallel/terraform_local/data_stream/local/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 000000000..cb3aea9e9 --- /dev/null +++ b/test/packages/parallel/terraform_local/data_stream/local/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,14 @@ +--- +description: Pipeline for processing sample logs +processors: +- json: + field: message + target_field: target +- set: + field: error.message + value: not present target + if: ctx?.target == null +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/test/packages/parallel/aws_logs/data_stream/generic/fields/base-fields.yml b/test/packages/parallel/terraform_local/data_stream/local/fields/base-fields.yml similarity index 55% rename from test/packages/parallel/aws_logs/data_stream/generic/fields/base-fields.yml rename to test/packages/parallel/terraform_local/data_stream/local/fields/base-fields.yml index c124b0964..7c798f453 100644 --- a/test/packages/parallel/aws_logs/data_stream/generic/fields/base-fields.yml +++ b/test/packages/parallel/terraform_local/data_stream/local/fields/base-fields.yml @@ -10,14 +10,3 @@ - name: '@timestamp' type: date description: Event timestamp. -- name: event.module - type: constant_keyword - description: Event module - value: aws -- name: event.dataset - type: constant_keyword - description: Event dataset - value: aws_logs.generic -- name: input.type - type: keyword - description: Type of Filebeat input. diff --git a/test/packages/parallel/terraform_local/data_stream/local/fields/ecs.yml b/test/packages/parallel/terraform_local/data_stream/local/fields/ecs.yml new file mode 100644 index 000000000..2d92bac2b --- /dev/null +++ b/test/packages/parallel/terraform_local/data_stream/local/fields/ecs.yml @@ -0,0 +1,8 @@ +- external: ecs + name: ecs.version +- external: ecs + name: log.file.path +- external: ecs + name: error.message +- external: ecs + name: message diff --git a/test/packages/parallel/terraform_local/data_stream/local/fields/fields.yml b/test/packages/parallel/terraform_local/data_stream/local/fields/fields.yml new file mode 100644 index 000000000..d665a691a --- /dev/null +++ b/test/packages/parallel/terraform_local/data_stream/local/fields/fields.yml @@ -0,0 +1,11 @@ +- name: input.type + type: keyword +- name: log.offset + type: long +- name: target + type: group + fields: + - name: contents + type: text + - name: file + type: keyword diff --git a/test/packages/parallel/terraform_local/data_stream/local/manifest.yml b/test/packages/parallel/terraform_local/data_stream/local/manifest.yml new file mode 100644 index 000000000..bd5af5c94 --- /dev/null +++ b/test/packages/parallel/terraform_local/data_stream/local/manifest.yml @@ -0,0 +1,17 @@ +title: "Local" +type: logs +streams: + - input: logfile + title: Sample logs + description: Collect sample logs + vars: + - name: paths + type: text + title: Paths + multi: true + default: + - /tmp/service_logs/*.log +elasticsearch: + index_template: + mappings: + subobjects: false diff --git a/test/packages/parallel/terraform_local/data_stream/outputs/_dev/deploy/tf/files/example.log b/test/packages/parallel/terraform_local/data_stream/outputs/_dev/deploy/tf/files/example.log new file mode 100644 index 000000000..8701353d8 --- /dev/null +++ b/test/packages/parallel/terraform_local/data_stream/outputs/_dev/deploy/tf/files/example.log @@ -0,0 +1 @@ +{ "contents": "Message from file", "file": "logs.json"} diff --git a/test/packages/parallel/terraform_local/data_stream/outputs/_dev/deploy/tf/main.tf b/test/packages/parallel/terraform_local/data_stream/outputs/_dev/deploy/tf/main.tf new file mode 100644 index 000000000..7ace607aa --- /dev/null +++ b/test/packages/parallel/terraform_local/data_stream/outputs/_dev/deploy/tf/main.tf @@ -0,0 +1,13 @@ +variable "TEST_RUN_ID" { + default = "detached" +} + +resource "local_file" "log" { + source = "./files/example.log" + filename = "/tmp/service_logs/${var.TEST_RUN_ID}.log" + file_permission = "0777" +} + +output "filename" { + value = local_file.log.filename +} diff --git a/test/packages/parallel/terraform_local/data_stream/outputs/_dev/test/system/test-default-config.yml b/test/packages/parallel/terraform_local/data_stream/outputs/_dev/test/system/test-default-config.yml new file mode 100644 index 000000000..e2fd5b513 --- /dev/null +++ b/test/packages/parallel/terraform_local/data_stream/outputs/_dev/test/system/test-default-config.yml @@ -0,0 +1,5 @@ +wait_for_data_timeout: 1m +data_stream: + vars: + paths: + - '{{TF_OUTPUT_filename}}' diff --git a/test/packages/parallel/terraform_local/data_stream/outputs/agent/stream/stream.yml.hbs b/test/packages/parallel/terraform_local/data_stream/outputs/agent/stream/stream.yml.hbs new file mode 100644 index 000000000..5845510de --- /dev/null +++ b/test/packages/parallel/terraform_local/data_stream/outputs/agent/stream/stream.yml.hbs @@ -0,0 +1,7 @@ +paths: +{{#each paths as |path i|}} + - {{path}} +{{/each}} +exclude_files: [".gz$"] +processors: + - add_locale: ~ diff --git a/test/packages/parallel/terraform_local/data_stream/outputs/elasticsearch/ingest_pipeline/default.yml b/test/packages/parallel/terraform_local/data_stream/outputs/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 000000000..cb3aea9e9 --- /dev/null +++ b/test/packages/parallel/terraform_local/data_stream/outputs/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,14 @@ +--- +description: Pipeline for processing sample logs +processors: +- json: + field: message + target_field: target +- set: + field: error.message + value: not present target + if: ctx?.target == null +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/test/packages/parallel/terraform_local/data_stream/outputs/fields/base-fields.yml b/test/packages/parallel/terraform_local/data_stream/outputs/fields/base-fields.yml new file mode 100644 index 000000000..7c798f453 --- /dev/null +++ b/test/packages/parallel/terraform_local/data_stream/outputs/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/test/packages/parallel/terraform_local/data_stream/outputs/fields/ecs.yml b/test/packages/parallel/terraform_local/data_stream/outputs/fields/ecs.yml new file mode 100644 index 000000000..2d92bac2b --- /dev/null +++ b/test/packages/parallel/terraform_local/data_stream/outputs/fields/ecs.yml @@ -0,0 +1,8 @@ +- external: ecs + name: ecs.version +- external: ecs + name: log.file.path +- external: ecs + name: error.message +- external: ecs + name: message diff --git a/test/packages/parallel/terraform_local/data_stream/outputs/fields/fields.yml b/test/packages/parallel/terraform_local/data_stream/outputs/fields/fields.yml new file mode 100644 index 000000000..d665a691a --- /dev/null +++ b/test/packages/parallel/terraform_local/data_stream/outputs/fields/fields.yml @@ -0,0 +1,11 @@ +- name: input.type + type: keyword +- name: log.offset + type: long +- name: target + type: group + fields: + - name: contents + type: text + - name: file + type: keyword diff --git a/test/packages/parallel/terraform_local/data_stream/outputs/manifest.yml b/test/packages/parallel/terraform_local/data_stream/outputs/manifest.yml new file mode 100644 index 000000000..dc8d182d1 --- /dev/null +++ b/test/packages/parallel/terraform_local/data_stream/outputs/manifest.yml @@ -0,0 +1,17 @@ +title: "Outputs" +type: logs +streams: + - input: logfile + title: Sample logs + description: Collect sample logs + vars: + - name: paths + type: text + title: Paths + multi: true + default: + - /other_path/*.log +elasticsearch: + index_template: + mappings: + subobjects: false diff --git a/test/packages/parallel/terraform_local/docs/README.md b/test/packages/parallel/terraform_local/docs/README.md new file mode 100644 index 000000000..28ff999ea --- /dev/null +++ b/test/packages/parallel/terraform_local/docs/README.md @@ -0,0 +1,84 @@ + + + +# Terraform Local + + + +## Data streams + + + + + + + + + + + +## Requirements + +You need Elasticsearch for storing and searching your data and Kibana for visualizing and managing it. +You can use our hosted Elasticsearch Service on Elastic Cloud, which is recommended, or self-manage the Elastic Stack on your own hardware. + + + +## Setup + + + +For step-by-step instructions on how to set up an integration, see the +[Getting started](https://www.elastic.co/guide/en/welcome-to-elastic/current/getting-started-observability.html) guide. + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/packages/parallel/terraform_local/img/sample-logo.svg b/test/packages/parallel/terraform_local/img/sample-logo.svg new file mode 100644 index 000000000..6268dd88f --- /dev/null +++ b/test/packages/parallel/terraform_local/img/sample-logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/test/packages/parallel/terraform_local/img/sample-screenshot.png b/test/packages/parallel/terraform_local/img/sample-screenshot.png new file mode 100644 index 000000000..d7a56a3ec Binary files /dev/null and b/test/packages/parallel/terraform_local/img/sample-screenshot.png differ diff --git a/test/packages/parallel/terraform_local/manifest.yml b/test/packages/parallel/terraform_local/manifest.yml new file mode 100644 index 000000000..4a5aa7ed7 --- /dev/null +++ b/test/packages/parallel/terraform_local/manifest.yml @@ -0,0 +1,36 @@ +format_version: 3.2.1 +name: terraform_local +title: "Terraform Local" +version: 0.0.1 +source: + license: "Elastic-2.0" +description: "Package to test creating local resources using terraform" +type: integration +categories: + - custom +conditions: + kibana: + version: "^8.14.3" + elastic: + subscription: "basic" +screenshots: + - src: /img/sample-screenshot.png + title: Sample screenshot + size: 600x600 + type: image/png +icons: + - src: /img/sample-logo.svg + title: Sample logo + size: 32x32 + type: image/svg+xml +policy_templates: + - name: sample + title: Sample logs + description: Collect sample logs + inputs: + - type: logfile + title: Collect sample logs from instances + description: Collecting sample logs +owner: + github: elastic/integrations + type: elastic