[O11y][AWS] Rally benchmark aws.cloudtrail (#9448)

elastic · Apr 29, 2024 · ace8fb4 · ace8fb4
1 parent d4e4aa4
commit ace8fb4
Show file tree

Hide file tree

Showing 4 changed files with 434 additions and 0 deletions.
diff --git a/packages/aws/_dev/benchmark/rally/cloudtrail-benchmark.yml b/packages/aws/_dev/benchmark/rally/cloudtrail-benchmark.yml
@@ -0,0 +1,14 @@
+---
+description: Benchmark of 20000 aws.cloudtrail events ingested
+data_stream:
+  name: cloudtrail
+corpora:
+  generator:
+    total_events: 20000
+    template:
+      type: gotext
+      path: ./cloudtrail-benchmark/template.ndjson
+    config:
+      path: ./cloudtrail-benchmark/config.yml
+    fields:
+      path: ./cloudtrail-benchmark/fields.yml
diff --git a/packages/aws/_dev/benchmark/rally/cloudtrail-benchmark/config.yml b/packages/aws/_dev/benchmark/rally/cloudtrail-benchmark/config.yml
@@ -0,0 +1,156 @@
+fields:
+  - name: timestamp
+    period: -24h
+  - name: digest_previous_s3_bucket
+    value: "alice-bucket"
+  - name: digest_s3_object
+    value: "AWSLogs/123456789123/CloudTrail-Digest/us-west-2/2020/09/11/123456789123_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T193649Z.json.gz"
+  - name: digest_public_Key_fingerprint
+    value: "47aaa19f7eec22e9bd0b5e58cfade8cb"
+  - name: digest_signature_algorithm
+    value: "SHA256withRSA"
+  - name: previous_digest_s3_object
+    value: "AWSLogs/123456789123/CloudTrail-Digest/us-west-2/2020/09/11/123456789123_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T183649Z.json.gz"
+  - name: previous_digest_hash_value
+    value: "531914fcfa0dbacf0c9dd1475a1fdcb5dea6e85921409f3c3ec0ba39063c860"
+  - name: digest_previous_hash_algorithm
+    value: "SHA-256"
+  - name: previous_digest_signature
+    value: "10e0872f32fa1d299d0cc98e94d4c88a6a2eada9d9fc3ae6d53dfe8d54c7caf807072f1e1eec47efdeecfcc22483887f8fddfc954ae587fba43e7676b5547f432fa8722ba1c5baa6b233bcb528ce7c01e3748aab8f28c16c024de79da820128b4c9e5ce65e98a9c4e631687ecc89c224a11bb3df06ce441ff740e4ac9fbd41159e77f5863550118284121f193e357866fbd0463faffb56e194af196e35a7675c3bbd0a398f43159343c3f59129d6339a281a8fdb3192f3fffea9bd21dbb0a705ebfae1921f2133aab0ad29522aea6df0828c1780d3f3ed6b8270ab3ba24459916b0fbbe82fba6ff9677bafe7306e0f5edcc0f1508cdb4e36f3e3b30e653e9987"
+  - name: eventCategory
+    value: "Insight"
+  - name: eventId
+    value: "11ea990b-4678-4bcd-8fbe-625EXAMPLE"
+  - name: userIdentity_accountId
+    range:
+      min: 1000000000000
+      max: 2000000000000
+  - name: userIdentity_type
+    enum: ["Root", "IAMUser", "AssumedRole", "Role", "FederatedUser", "AWSAccount"]
+  - name: userIdentity_arn
+    value: "arn:aws:iam::123456789012:user/Alice"
+  - name: userIdentity_eventSource
+    value: "iam.amazonaws.com"
+  - name: userIdentity_eventName
+    value: "UpdateUser"
+  - name: userIdentity_eventTime
+    value: "2014-07-08T17:35:27Z"
+  - name: newestEventTime
+    value: "2020-09-11T19:26:24.000Z"
+  - name: oldestEventTime
+    value: "2020-09-11T18:32:04.000Z"
+  - name: userIdentity_awsregion
+    enum: ["us-east-1", "us-east-2", "us-west-1", "us-west-2", "ap-south-1", "ap-northeast-3", "ap-northeast-2", "ap-southeast-1", "ap-southeast-2", "ap-northeast-1", "ca-central-1", "eu-central-1", "eu-west-1", "eu-west-2", "eu-west-3", "eu-north-1", "sa-east-1", "af-south-1", "ap-east-1", "ap-south-2", "ap-southeast-3", "eu-south-2", "eu-central-2", "me-south-1", "me-central-1"]
+    cardinality: 25
+  - name: userIdentity_sourceIPAddress
+    value: 127.0.0.1
+  - name: userIdentity_useragent
+    value: "aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46"
+  - name: requestID
+    value: "3a6b3260-739d-465e-9406-bcEXAMPLE"
+  - name: event_id
+    value: "9150d546-3564-4262-8e62-110EXAMPLE"
+  - name: logFiles
+    value: "https://elastic-package-aws-bucket-64547.s3.us-east-1.amazonaws.com/cloudtrail-digest.log"
+  - name: eventType
+    value: "AwsApiCall"
+  - name: userIdentity_requestparameters_username
+    value: "username"
+  - name: userIdentity_requestparameters_newusername
+    value: "newusername"
+  - name: recipientAccountId
+    value: "123456789012"
+  - name: digest_start_time
+    value: "2020-09-11T18:36:49.000Z"
+  - name: userIdentity_sessionContext_mfaauthenticated
+    enum: ["true", "false"]
+  - name: userIdentity_sessionContext_creationDate
+    value: "2020-09-11T18:12:52.000Z"
+  - name: invokedBy
+    value: "signin.amazonaws.com"
+  - name: sharedEventId
+    value: "123456789012"
+  - name: vpcEndpointId
+    value: "123456789012"
+  - name: userIdentity_sessionContext_sessionIssuer_type
+    enum: ["Root", "IAMUser", "Role"]
+  - name: userIdentity_sessionContext_sessionIssuer_principalId
+    value: "AROAIDPPEZS35WEXAMPLE"
+  - name: userIdentity_sessionContext_sessionIssuer_arn
+    value: "arn:aws:iam::123456789012:role/RoleToBeAssumed"
+  - name: userIdentity_sessionContext_sessionIssuer_accountId
+    range:
+      min: 1000000000000
+      max: 2000000000000
+  - name: userIdentity_sessionContext_sessionIssuer_userName
+    value: "RoleToBeAssumed"
+  - name: errorMessage
+    value: "Failed authentication"
+  - name: errorCode
+    value: "AccessDeniedException"
+  - name: apiVersion
+    value: "api1.1"
+  - name: responseElements_ConsoleLogin
+    enum: ["Failure", "Success"]
+  - name: additionalEventData_MobileVersion
+    enum: ["Yes", "No"]
+  - name: additionalEventData_LoginTo
+    value: "https://console.aws.amazon.com/sns"
+  - name: additionalEventData_MFAUsed
+    enum: ["Yes", "No"]
+  - name: aws_Account_Id
+    value: "hdbcskndcl123y2873y"
+  - name: digest_start_time
+    value: "2020-09-11T18:36:49Z"
+  - name: digest_end_time
+    value: "2020-09-12T19:13:56Z"
+  - name: digest_s3_bucket
+    value: "alice-bucket"
+  - name: resources_type
+    value: "AWS::IAM::Role"
+  - name: resources_ARN
+    value: "arn:aws:iam::111122223333:role/JohnRole2"
+  - name: resources_accountId
+    value: "111111100000011111"
+  - name: readOnly
+    enum: ["true", "false"]
+  - name: managementEvent
+    value: "cloudtrail event is genearted"
+  - name: insightDetails_state
+    enum: ["End", "start"]
+  - name: insightDetails_eventSource
+    value: "iam.amazonaws.com"
+  - name: insightDetails_eventName
+    value: "AttachUserPolicy"
+  - name: insightDetails_insightType
+    value: "ApiCallRateInsight"
+  - name: insightDetails_insffightContext_statistics_baseline_average
+    range:
+      min: 1
+      max: 1000
+  - name: insightDetails_insffightContext_statistics_insight_average
+    range:
+      min: 1
+      max: 1000
+  - name: insightDetails_insffightContext_statistics_insightDuration
+    range:
+      min: 1
+      max: 100
+  - name: insightDetails_insffightContext_statistics_baselineDuration
+    range:
+      min: 1
+      max: 100000
+  - name: insightDetails_insffightContext_attributions_attribute
+    value: "userIdentityArn"
+  - name: insightDetails_insffightContext_attributions_insight_value
+    value: "arn:aws:iam::123456789012:user/Alice"
+  - name: insightDetails_insffightContext_attributions_insight_average
+    range:
+      min: 1
+      max: 100
+  - name: insightDetails_insffightContext_attributions_baseline
+    value: []
+  - name: bucket_num
+    range:
+      min: 63461
+      max: 63471
diff --git a/packages/aws/_dev/benchmark/rally/cloudtrail-benchmark/fields.yml b/packages/aws/_dev/benchmark/rally/cloudtrail-benchmark/fields.yml
@@ -0,0 +1,148 @@
+- name: timestamp
+  type: date
+- name: aws_Account_Id
+  type: keyword
+- name: file_hash_sha256
+  type: keyword
+- name: file_path
+  type: keyword
+- name: digest_start_time
+  type: date
+- name: digest_end_time
+  type: date
+- name: digest_previous_s3_bucket
+  type: keyword
+- name: digest_s3_bucket
+  type: keyword
+- name: digest_s3_object
+  type: keyword
+- name: digest_public_Key_fingerprint
+  type: keyword
+- name: digest_signature_algorithm
+  type: keyword
+- name: newestEventTime
+  type: date
+- name: oldestEventTime
+  type: date
+- name: previous_digest_s3_object
+  type: keyword
+- name: previous_digest_hash_value
+  type: keyword
+- name: digest_previous_hash_algorithm
+  type: keyword
+- name: logFiles
+  type: keyword
+- name: previous_digest_signature
+  type: keyword
+- name: eventCategory
+  type: keyword
+- name: eventId
+  type: keyword
+- name: userIdentity_type
+  type: keyword
+- name: userIdentity_principalId
+  type: keyword
+- name: userIdentity_arn
+  type: keyword
+- name: userIdentity_accountId
+  type: long
+- name: userIdentity_accesskeyId
+  type: keyword
+  example: example_key
+- name: userIdentity_userName
+  type: keyword
+- name: userIdentity_eventTime
+  type: date
+- name: userIdentity_eventSource
+  type: keyword
+- name: userIdentity_eventName
+  type: keyword
+- name: userIdentity_awsregion
+  type: keyword
+- name: userIdentity_sourceIPAddress
+  type: ip
+- name: userIdentity_useragent
+  type: keyword
+- name: userIdentity_requestparameters_username
+  type: keyword
+- name: userIdentity_requestparameters_newusername
+  type: keyword
+- name: requestID
+  type: keyword
+- name: eventType
+  type: keyword
+- name: recipientAccountId
+  type: keyword
+- name: userIdentity_sessionContext_mfaauthenticated
+  type: boolean
+- name: userIdentity_sessionContext_creationDate
+  type: date
+- name: sharedEventId
+  type: keyword
+- name: vpcEndpointId
+  type: keyword
+- name: invokedBy
+  type: keyword
+- name: userIdentity_sessionContext_sessionIssuer_type
+  type: keyword
+- name: userIdentity_sessionContext_sessionIssuer_principalId
+  type: keyword
+- name: userIdentity_sessionContext_sessionIssuer_arn
+  type: keyword
+- name: userIdentity_sessionContext_sessionIssuer_accountId
+  type: keyword
+- name: userIdentity_sessionContext_sessionIssuer_userName
+  type: keyword
+- name: errorMessage
+  type: keyword
+- name: errorCode
+  type: keyword
+- name: apiVersion
+  type: keyword
+- name: responseElements_ConsoleLogin
+  type: keyword
+- name: additionalEventData_MobileVersion
+  type: boolean
+- name: additionalEventData_LoginTo
+  type: keyword
+- name: additionalEventData_MFAUsed
+  type: boolean
+- name: resources_type
+  type: keyword
+- name: resources_ARN
+  type: keyword
+- name: resources_accountId
+  type: keyword
+- name: readOnly
+  type: boolean
+- name: managementEvent
+  type: keyword
+- name: insightDetails_state
+  type: boolean
+- name: insightDetails_eventSource
+  type: "iam.amazonaws.com"
+- name: insightDetails_eventName
+  type: "AttachUserPolicy"
+- name: insightDetails_insightType
+  type: "ApiCallRateInsight"
+- name: insightDetails_insffightContext_statistics_baseline_average
+  type: float
+- name: insightDetails_insffightContext_statistics_insight_average
+  type: float
+- name: insightDetails_insffightContext_statistics_insightDuration
+  type: long
+- name: insightDetails_insffightContext_statistics_baselineDuration
+  type: long
+- name: insightDetails_insffightContext_attributions_attribute
+  type: keyword
+- name: insightDetails_insffightContext_attributions_insight_value
+  type: keyword
+- name: insightDetails_insffightContext_attributions_insight_average
+  type: float
+- name: insightDetails_insffightContext_attributions_baseline
+  type: keyword
+- name: file_name
+  type: keyword
+  example: extra-samples
+- name: bucket_num
+  type: long