From bd3762f9a1b7e384aee35a8e70f3a2ddfbf6650a Mon Sep 17 00:00:00 2001 From: aliabbas-elastic Date: Tue, 26 Mar 2024 19:43:53 +0530 Subject: [PATCH 1/2] rally benchmark aws.cloudtrail --- .../benchmark/rally/cloudtrail-benchmark.yml | 14 ++ .../rally/cloudtrail-benchmark/config.yml | 154 ++++++++++++++++++ .../rally/cloudtrail-benchmark/fields.yml | 148 +++++++++++++++++ .../cloudtrail-benchmark/template.ndjson | 116 +++++++++++++ 4 files changed, 432 insertions(+) create mode 100644 packages/aws/_dev/benchmark/rally/cloudtrail-benchmark.yml create mode 100644 packages/aws/_dev/benchmark/rally/cloudtrail-benchmark/config.yml create mode 100644 packages/aws/_dev/benchmark/rally/cloudtrail-benchmark/fields.yml create mode 100644 packages/aws/_dev/benchmark/rally/cloudtrail-benchmark/template.ndjson diff --git a/packages/aws/_dev/benchmark/rally/cloudtrail-benchmark.yml b/packages/aws/_dev/benchmark/rally/cloudtrail-benchmark.yml new file mode 100644 index 00000000000..4f103e193e4 --- /dev/null +++ b/packages/aws/_dev/benchmark/rally/cloudtrail-benchmark.yml @@ -0,0 +1,14 @@ +--- +description: Benchmark of 20000 aws.cloudtrail events ingested +data_stream: + name: cloudtrail +corpora: + generator: + total_events: 20000 + template: + type: gotext + path: ./cloudtrail-benchmark/template.ndjson + config: + path: ./cloudtrail-benchmark/config.yml + fields: + path: ./cloudtrail-benchmark/fields.yml diff --git a/packages/aws/_dev/benchmark/rally/cloudtrail-benchmark/config.yml b/packages/aws/_dev/benchmark/rally/cloudtrail-benchmark/config.yml new file mode 100644 index 00000000000..795e0658889 --- /dev/null +++ b/packages/aws/_dev/benchmark/rally/cloudtrail-benchmark/config.yml @@ -0,0 +1,154 @@ +fields: + - name: timestamp + period: -24h + - name: digest_previous_s3_bucket + value: "alice-bucket" + - name: digest_s3_object + value: "AWSLogs/123456789123/CloudTrail-Digest/us-west-2/2020/09/11/123456789123_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T193649Z.json.gz" + - name: digest_public_Key_fingerprint + value: "47aaa19f7eec22e9bd0b5e58cfade8cb" + - name: digest_signature_algorithm + value: "SHA256withRSA" + - name: previous_digest_s3_object + value: "AWSLogs/123456789123/CloudTrail-Digest/us-west-2/2020/09/11/123456789123_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T183649Z.json.gz" + - name: previous_digest_hash_value + value: "531914fcfa0dbacf0c9dd1475a1fdcb5dea6e85921409f3c3ec0ba39063c860" + - name: digest_previous_hash_algorithm + value: "SHA-256" + - name: previous_digest_signature + value: "10e0872f32fa1d299d0cc98e94d4c88a6a2eada9d9fc3ae6d53dfe8d54c7caf807072f1e1eec47efdeecfcc22483887f8fddfc954ae587fba43e7676b5547f432fa8722ba1c5baa6b233bcb528ce7c01e3748aab8f28c16c024de79da820128b4c9e5ce65e98a9c4e631687ecc89c224a11bb3df06ce441ff740e4ac9fbd41159e77f5863550118284121f193e357866fbd0463faffb56e194af196e35a7675c3bbd0a398f43159343c3f59129d6339a281a8fdb3192f3fffea9bd21dbb0a705ebfae1921f2133aab0ad29522aea6df0828c1780d3f3ed6b8270ab3ba24459916b0fbbe82fba6ff9677bafe7306e0f5edcc0f1508cdb4e36f3e3b30e653e9987" + - name: eventCategory + value: "Insight" + - name: eventId + value: "11ea990b-4678-4bcd-8fbe-625EXAMPLE" + - name: userIdentity_accountId + range: + min: 1000000000000 + max: 2000000000000 + - name: userIdentity_type + enum: ["Root", "IAMUser", "AssumedRole", "Role", "FederatedUser", "AWSAccount"] + - name: userIdentity_arn + value: "arn:aws:iam::123456789012:user/Alice" + - name: userIdentity_eventSource + value: "iam.amazonaws.com" + - name: userIdentity_eventName + value: "UpdateUser" + - name: userIdentity_eventTime + value: "2014-07-08T17:35:27Z" + - name: newestEventTime + value: "2020-09-11T19:26:24.000Z" + - name: oldestEventTime + value: "2020-09-11T18:32:04.000Z" + - name: userIdentity_awsregion + enum: ["us-east-1", "us-east-2", "us-west-1", "us-west-2", "ap-south-1", "ap-northeast-3", "ap-northeast-2", "ap-southeast-1", "ap-southeast-2", "ap-northeast-1", "ca-central-1", "eu-central-1", "eu-west-1", "eu-west-2", "eu-west-3", "eu-north-1", "sa-east-1", "af-south-1", "ap-east-1", "ap-south-2", "ap-southeast-3", "eu-south-2", "eu-central-2", "me-south-1", "me-central-1"] + cardinality: 25 + - name: userIdentity_sourceIPAddress + value: 127.0.0.1 + - name: userIdentity_useragent + value: "aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46" + - name: requestID + value: "3a6b3260-739d-465e-9406-bcEXAMPLE" + - name: event_id + value: "9150d546-3564-4262-8e62-110EXAMPLE" + - name: logFiles + value: "https://elastic-package-aws-bucket-64547.s3.us-east-1.amazonaws.com/cloudtrail-digest.log" + - name: eventType + value: "AwsApiCall" + - name: userIdentity_requestparameters_username + value: "username" + - name: userIdentity_requestparameters_newusername + value: "newusername" + - name: recipientAccountId + value: "123456789012" + - name: digest_start_time + value: "2020-09-11T18:36:49.000Z" + - name: userIdentity_sessionContext_mfaauthenticated + enum: ["true", "false"] + - name: userIdentity_sessionContext_creationDate + value: "2020-09-11T18:12:52.000Z" + - name: invokedBy + value: "signin.amazonaws.com" + - name: sharedEventId + value: "123456789012" + - name: vpcEndpointId + value: "123456789012" + - name: userIdentity_sessionContext_sessionIssuer_type + enum: ["Root", "IAMUser", "Role"] + - name: userIdentity_sessionContext_sessionIssuer_principalId + value: "AROAIDPPEZS35WEXAMPLE" + - name: userIdentity_sessionContext_sessionIssuer_arn + value: "arn:aws:iam::123456789012:role/RoleToBeAssumed" + - name: userIdentity_sessionContext_sessionIssuer_accountId + value: "123456789044444412" + - name: userIdentity_sessionContext_sessionIssuer_userName + value: "RoleToBeAssumed" + - name: errorMessage + value: "Failed authentication" + - name: errorCode + value: "AccessDeniedException" + - name: apiVersion + value: "api1.1" + - name: responseElements_ConsoleLogin + enum: ["Failure", "Success"] + - name: additionalEventData_MobileVersion + enum: ["Yes", "No"] + - name: additionalEventData_LoginTo + value: "https://console.aws.amazon.com/sns" + - name: additionalEventData_MFAUsed + enum: ["Yes", "No"] + - name: aws_Account_Id + value: "hdbcskndcl123y2873y" + - name: digest_start_time + value: "2020-09-11T18:36:49Z" + - name: digest_end_time + value: "2020-09-12T19:13:56Z" + - name: digest_s3_bucket + value: "alice-bucket" + - name: resources_type + value: "AWS::IAM::Role" + - name: resources_ARN + value: "arn:aws:iam::111122223333:role/JohnRole2" + - name: resources_accountId + value: "111111100000011111" + - name: readOnly + enum: ["true", "false"] + - name: managementEvent + value: "cloudtrail event is genearted" + - name: insightDetails_state + enum: ["End", "start"] + - name: insightDetails_eventSource + value: "iam.amazonaws.com" + - name: insightDetails_eventName + value: "AttachUserPolicy" + - name: insightDetails_insightType + value: "ApiCallRateInsight" + - name: insightDetails_insffightContext_statistics_baseline_average + range: + min: 1 + max: 1000 + - name: insightDetails_insffightContext_statistics_insight_average + range: + min: 1 + max: 1000 + - name: insightDetails_insffightContext_statistics_insightDuration + range: + min: 1 + max: 100 + - name: insightDetails_insffightContext_statistics_baselineDuration + range: + min: 1 + max: 100000 + - name: insightDetails_insffightContext_attributions_attribute + value: "userIdentityArn" + - name: insightDetails_insffightContext_attributions_insight_value + value: "arn:aws:iam::123456789012:user/Alice" + - name: insightDetails_insffightContext_attributions_insight_average + range: + min: 1 + max: 100 + - name: insightDetails_insffightContext_attributions_baseline + value: [] + - name: bucket_num + range: + min: 63461 + max: 63471 diff --git a/packages/aws/_dev/benchmark/rally/cloudtrail-benchmark/fields.yml b/packages/aws/_dev/benchmark/rally/cloudtrail-benchmark/fields.yml new file mode 100644 index 00000000000..0cc86ab86c9 --- /dev/null +++ b/packages/aws/_dev/benchmark/rally/cloudtrail-benchmark/fields.yml @@ -0,0 +1,148 @@ +- name: timestamp + type: date +- name: aws_Account_Id + type: keyword +- name: file_hash_sha256 + type: keyword +- name: file_path + type: keyword +- name: digest_start_time + type: date +- name: digest_end_time + type: date +- name: digest_previous_s3_bucket + type: keyword +- name: digest_s3_bucket + type: keyword +- name: digest_s3_object + type: keyword +- name: digest_public_Key_fingerprint + type: keyword +- name: digest_signature_algorithm + type: keyword +- name: newestEventTime + type: date +- name: oldestEventTime + type: date +- name: previous_digest_s3_object + type: keyword +- name: previous_digest_hash_value + type: keyword +- name: digest_previous_hash_algorithm + type: keyword +- name: logFiles + type: keyword +- name: previous_digest_signature + type: keyword +- name: eventCategory + type: keyword +- name: eventId + type: keyword +- name: userIdentity_type + type: keyword +- name: userIdentity_principalId + type: keyword +- name: userIdentity_arn + type: keyword +- name: userIdentity_accountId + type: long +- name: userIdentity_accesskeyId + type: keyword + example: example_key +- name: userIdentity_userName + type: keyword +- name: userIdentity_eventTime + type: date +- name: userIdentity_eventSource + type: keyword +- name: userIdentity_eventName + type: keyword +- name: userIdentity_awsregion + type: keyword +- name: userIdentity_sourceIPAddress + type: ip +- name: userIdentity_useragent + type: keyword +- name: userIdentity_requestparameters_username + type: keyword +- name: userIdentity_requestparameters_newusername + type: keyword +- name: requestID + type: keyword +- name: eventType + type: keyword +- name: recipientAccountId + type: keyword +- name: userIdentity_sessionContext_mfaauthenticated + type: boolean +- name: userIdentity_sessionContext_creationDate + type: date +- name: sharedEventId + type: keyword +- name: vpcEndpointId + type: keyword +- name: invokedBy + type: keyword +- name: userIdentity_sessionContext_sessionIssuer_type + type: keyword +- name: userIdentity_sessionContext_sessionIssuer_principalId + type: keyword +- name: userIdentity_sessionContext_sessionIssuer_arn + type: keyword +- name: userIdentity_sessionContext_sessionIssuer_accountId + type: keyword +- name: userIdentity_sessionContext_sessionIssuer_userName + type: keyword +- name: errorMessage + type: keyword +- name: errorCode + type: keyword +- name: apiVersion + type: keyword +- name: responseElements_ConsoleLogin + type: keyword +- name: additionalEventData_MobileVersion + type: boolean +- name: additionalEventData_LoginTo + type: keyword +- name: additionalEventData_MFAUsed + type: boolean +- name: resources_type + type: keyword +- name: resources_ARN + type: keyword +- name: resources_accountId + type: keyword +- name: readOnly + type: boolean +- name: managementEvent + type: keyword +- name: insightDetails_state + type: boolean +- name: insightDetails_eventSource + type: "iam.amazonaws.com" +- name: insightDetails_eventName + type: "AttachUserPolicy" +- name: insightDetails_insightType + type: "ApiCallRateInsight" +- name: insightDetails_insffightContext_statistics_baseline_average + type: float +- name: insightDetails_insffightContext_statistics_insight_average + type: float +- name: insightDetails_insffightContext_statistics_insightDuration + type: long +- name: insightDetails_insffightContext_statistics_baselineDuration + type: long +- name: insightDetails_insffightContext_attributions_attribute + type: keyword +- name: insightDetails_insffightContext_attributions_insight_value + type: keyword +- name: insightDetails_insffightContext_attributions_insight_average + type: float +- name: insightDetails_insffightContext_attributions_baseline + type: keyword +- name: file_name + type: keyword + example: extra-samples +- name: bucket_num + type: long diff --git a/packages/aws/_dev/benchmark/rally/cloudtrail-benchmark/template.ndjson b/packages/aws/_dev/benchmark/rally/cloudtrail-benchmark/template.ndjson new file mode 100644 index 00000000000..37729917652 --- /dev/null +++ b/packages/aws/_dev/benchmark/rally/cloudtrail-benchmark/template.ndjson @@ -0,0 +1,116 @@ +{{- $timestamp := generate "timestamp" }} +{{- $aws_Account_Id := generate "aws_Account_Id" }} +{{- $digest_start_time := generate "digest_start_time" }} +{{- $digest_end_time := generate "digest_end_time" }} +{{- $digest_previous_s3_bucket := generate "digest_previous_s3_bucket" }} +{{- $digest_s3_bucket := generate "digest_s3_bucket" }} +{{- $digest_s3_object := generate "digest_s3_object" }} +{{- $digest_public_Key_fingerprint := generate "digest_public_Key_fingerprint" }} +{{- $digest_signature_algorithm := generate "digest_signature_algorithm" }} +{{- $newestEventTime := generate "newestEventTime" }} +{{- $oldestEventTime := generate "oldestEventTime" }} +{{- $previous_digest_s3_object := generate "previous_digest_s3_object" }} +{{- $previous_digest_hash_value := generate "previous_digest_hash_value" }} +{{- $digest_previous_hash_algorithm := generate "digest_previous_hash_algorithm" }} +{{- $previous_digest_signature := generate "previous_digest_signature" }} +{{- $eventCategory := generate "eventCategory" }} +{{- $eventId := generate "eventId" }} +{{- $userIdentity_type := generate "userIdentity_type" }} +{{- $userIdentity_principalId := generate "userIdentity_principalId" }} +{{- $userIdentity_arn := generate "userIdentity_arn" }} +{{- $userIdentity_accountId := generate "userIdentity_accountId" }} +{{- $userIdentity_accesskeyId := generate "userIdentity_accesskeyId" }} +{{- $userIdentity_userName := generate "userIdentity_userName" }} +{{- $userIdentity_eventTime := generate "userIdentity_eventTime" }} +{{- $userIdentity_eventSource := generate "userIdentity_eventSource" }} +{{- $userIdentity_eventName := generate "userIdentity_eventName" }} +{{- $userIdentity_awsregion := generate "userIdentity_awsregion" }} +{{- $userIdentity_sourceIPAddress := generate "userIdentity_sourceIPAddress" }} +{{- $userIdentity_useragent := generate "userIdentity_useragent" }} +{{- $userIdentity_requestparameters_username := generate "userIdentity_requestparameters_username" }} +{{- $userIdentity_requestparameters_newusername := generate "userIdentity_requestparameters_newusername" }} +{{- $requestID := generate "requestID" }} +{{- $eventType := generate "eventType" }} +{{- $recipientAccountId := generate "recipientAccountId" }} +{{- $userIdentity_sessionContext_mfaauthenticated := generate "userIdentity_sessionContext_mfaauthenticated" }} +{{- $userIdentity_sessionContext_creationDate := generate "userIdentity_sessionContext_creationDate" }} +{{- $invokedBy := generate "invokedBy" }} +{{- $sharedEventId := generate "sharedEventId" }} +{{- $vpcEndpointId := generate "vpcEndpointId" }} +{{- $userIdentity_sessionContext_sessionIssuer_type := generate "userIdentity_sessionContext_sessionIssuer_type" }} +{{- $userIdentity_sessionContext_sessionIssuer_principalId := generate "userIdentity_sessionContext_sessionIssuer_principalId" }} +{{- $userIdentity_sessionContext_sessionIssuer_arn := generate "userIdentity_sessionContext_sessionIssuer_arn" }} +{{- $userIdentity_sessionContext_sessionIssuer_accountId := generate "userIdentity_sessionContext_sessionIssuer_accountId" }} +{{- $userIdentity_sessionContext_sessionIssuer_userName := generate "userIdentity_sessionContext_sessionIssuer_userName" }} +{{- $errorMessage := generate "errorMessage" }} +{{- $errorCode := generate "errorCode" }} +{{- $apiVersion := generate "apiVersion" }} +{{- $responseElements_ConsoleLogin := generate "responseElements_ConsoleLogin" }} +{{- $additionalEventData_MobileVersion := generate "additionalEventData_MobileVersion" }} +{{- $additionalEventData_LoginTo := generate "additionalEventData_LoginTo" }} +{{- $additionalEventData_MFAUsed := generate "additionalEventData_MFAUsed" }} +{{- $resources_accountId := generate "resources_accountId" }} +{{- $resources_ARN := generate "resources_ARN" }} +{{- $resources_type := generate "resources_type" }} +{{- $readOnly := generate "readOnly" }} +{{- $logFiles := generate "logFiles" }} +{{- $managementEvent := generate "managementEvent" }} +{{- $insightDetails_state := generate "insightDetails_state" }} +{{- $insightDetails_eventSource := generate "insightDetails_eventSource" }} +{{- $insightDetails_eventName := generate "insightDetails_eventName" }} +{{- $insightDetails_insightType := generate "insightDetails_insightType" }} +{{- $insightDetails_insffightContext_statistics_baseline_average := generate "insightDetails_insffightContext_statistics_baseline_average" }} +{{- $insightDetails_insffightContext_attributions_attribute := generate "insightDetails_insffightContext_attributions_attribute" }} +{{- $insightDetails_insffightContext_attributions_insight_value := generate "insightDetails_insffightContext_attributions_insight_value" }} +{{- $insightDetails_insffightContext_attributions_insight_average := generate "insightDetails_insffightContext_attributions_insight_average" }} +{{- $insightDetails_insffightContext_attributions_baseline := generate "insightDetails_insffightContext_attributions_baseline" }} +{{- $insightDetails_insffightContext_statistics_insight_average := generate "insightDetails_insffightContext_statistics_insight_average" }} +{{- $insightDetails_insffightContext_statistics_insightDuration := generate "insightDetails_insffightContext_statistics_insightDuration" }} +{{- $insightDetails_insffightContext_statistics_baselineDuration := generate "insightDetails_insffightContext_statistics_baselineDuration" }} +{{- $bucket_num := generate "bucket_num" }} +{{- $file_name := generate "file_name" }} +{ + "@timestamp": "{{ $timestamp.Format "2006-01-02T15:04:05.000Z07:00" }}", + "agent": { + "ephemeral_id": "22ed892c-43bd-408a-9121-65e2f5b6a56e", + "id": "de42127b-4db8-4471-824e-a7b14f478663", + "name": "aws-scale-123456", + "type": "filebeat", + "version": "8.8.0" + }, + "s3": { + "bucket": { + "arn": "arn:aws:s3:::elastic-package-aws-bucket-{{ $bucket_num }}", + "name": "elastic-package-aws-bucket-{{ $bucket_num }}" + }, + "object": { + "key": "{{ $file_name }}.log" + } + }, + "data_stream": { + "dataset": "aws.cloudtrail", + "namespace": "ep", + "type": "logs" + }, + "ecs": { + "version": "8.8.0" + }, + "elastic_agent": { + "id": "de42127b-4db8-4471-824e-a7b14f478663", + "snapshot": false, + "version": "8.8.0" + }, + "message": "{ \"eventVersion\": \"1.05\", \"userIdentity\": { \"type\": \"IAMUser\", \"principalId\": \"EXAMPLE_ID\", \"arn\": \"arn:aws:iam::0123456789012:user/Alice\", \"accountId\": \"0123456789012\", \"accessKeyId\": \"EXAMPLE_KEY\", \"userName\": \"Alice\", \"sessionContext\": { \"attributes\": { \"mfaAuthenticated\": \"true\", \"creationDate\": \"2020-01-10T14:38:30Z\" }, \"sessionIssuer\": { \"accountId\": \"111111111111\", \"arn\": \"arn:aws:iam::111111111111:role/JohnRole1\", \"principalId\": \"AROAIN5ATK5U7KEXAMPLE\", \"type\": \"Role\" } }, \"invokedBy\": \"signin.amazonaws.com\" }, \"eventTime\": \"2020-01-10T16:06:40Z\", \"eventSource\": \"iam.amazonaws.com\", \"eventName\": \"UploadSSHPublicKey\", \"awsRegion\": \"us-east-1\", \"sourceIPAddress\": \"127.0.0.1\", \"userAgent\": \"signin.amazonaws.com\", \"requestParameters\": { \"sSHPublicKeyBody\": \"ssh-rsa AAAAdeadcodedeadcode Alice@localhost.domain\", \"userName\": \"Alice\" }, \"responseElements\": { \"sSHPublicKey\": { \"fingerprint\": \"de:ad:c0:de:de:ad:c0:de:de:ad:c0:de:de:ad:c0:de\", \"status\": \"Active\", \"uploadDate\": \"Jan 10, 2020 4:06:40 PM\", \"userName\": \"Alice\", \"sSHPublicKeyId\": \"EXAMPLE_KEY_ID\", \"sSHPublicKeyBody\": \"ssh-rsa AAAAdeadcodedeadcode Alice@localhost.domain\" } }, \"requestID\": \"EXAMPLE-44b9-41cd-90f2-EXAMPLE\", \"eventID\": \"EXAMPLE-9a9d-4da4-9998-EXAMPLE\", \"eventType\": \"AwsApiCall\", \"recipientAccountId\": \"0123456789012\" }", + "event": { + "dataset": "aws.cloudtrail" + }, + "input": { + "type": "aws-s3" + }, + "offset": 0, + "tags": [ + "preserve_original_event", + "forwarded", + "aws-cloudtrail" + ] +} \ No newline at end of file From 84b5c355e272bdb28057e3dad3fe790c7cc06cca Mon Sep 17 00:00:00 2001 From: aliabbas-elastic Date: Thu, 4 Apr 2024 13:02:22 +0530 Subject: [PATCH 2/2] update range --- .../aws/_dev/benchmark/rally/cloudtrail-benchmark/config.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/aws/_dev/benchmark/rally/cloudtrail-benchmark/config.yml b/packages/aws/_dev/benchmark/rally/cloudtrail-benchmark/config.yml index 795e0658889..5a451e23600 100644 --- a/packages/aws/_dev/benchmark/rally/cloudtrail-benchmark/config.yml +++ b/packages/aws/_dev/benchmark/rally/cloudtrail-benchmark/config.yml @@ -79,7 +79,9 @@ fields: - name: userIdentity_sessionContext_sessionIssuer_arn value: "arn:aws:iam::123456789012:role/RoleToBeAssumed" - name: userIdentity_sessionContext_sessionIssuer_accountId - value: "123456789044444412" + range: + min: 1000000000000 + max: 2000000000000 - name: userIdentity_sessionContext_sessionIssuer_userName value: "RoleToBeAssumed" - name: errorMessage