From 47f0f0e3eeccb6ff0328b7529d11401634805a78 Mon Sep 17 00:00:00 2001 From: Mohanna Shahrad Date: Fri, 26 Aug 2022 23:34:49 +0000 Subject: [PATCH] Added the new opensearch branch - no conflict with timestream --- .../aws_cdk/OpenSearchPattern/.gitignore | 9 + .../aws_cdk/OpenSearchPattern/README.md | 154 +++ .../aws_cdk/OpenSearchPattern/app.py | 27 + .../aws_cdk/OpenSearchPattern/cdk.json | 52 + .../open_search_pattern/__init__.py | 0 .../open_search_pattern_stack.py | 369 ++++++ .../OpenSearchPattern/requirements-dev.txt | 1 + .../OpenSearchPattern/requirements.txt | 2 + .../aws_cdk/OpenSearchPattern/source.bat | 13 + .../OpenSearchPattern/tests/__init__.py | 0 .../OpenSearchPattern/tests/unit/__init__.py | 0 .../unit/test_open_search_pattern_stack.py | 798 +++++++++++++ .../demo_templates/opensearch_pattern.json | 1003 +++++++++++++++++ .../user_guides/opensearch_guide.md | 202 ++++ 14 files changed, 2630 insertions(+) create mode 100644 cloud_templates/aws_cdk/OpenSearchPattern/.gitignore create mode 100644 cloud_templates/aws_cdk/OpenSearchPattern/README.md create mode 100644 cloud_templates/aws_cdk/OpenSearchPattern/app.py create mode 100644 cloud_templates/aws_cdk/OpenSearchPattern/cdk.json create mode 100644 cloud_templates/aws_cdk/OpenSearchPattern/open_search_pattern/__init__.py create mode 100644 cloud_templates/aws_cdk/OpenSearchPattern/open_search_pattern/open_search_pattern_stack.py create mode 100644 cloud_templates/aws_cdk/OpenSearchPattern/requirements-dev.txt create mode 100644 cloud_templates/aws_cdk/OpenSearchPattern/requirements.txt create mode 100644 cloud_templates/aws_cdk/OpenSearchPattern/source.bat create mode 100644 cloud_templates/aws_cdk/OpenSearchPattern/tests/__init__.py create mode 100644 cloud_templates/aws_cdk/OpenSearchPattern/tests/unit/__init__.py create mode 100644 cloud_templates/aws_cdk/OpenSearchPattern/tests/unit/test_open_search_pattern_stack.py create mode 100644 cloud_templates/demo/demo_templates/opensearch_pattern.json create mode 100644 cloud_templates/user_guides/opensearch_guide.md diff --git a/cloud_templates/aws_cdk/OpenSearchPattern/.gitignore b/cloud_templates/aws_cdk/OpenSearchPattern/.gitignore new file mode 100644 index 0000000..3037faa --- /dev/null +++ b/cloud_templates/aws_cdk/OpenSearchPattern/.gitignore @@ -0,0 +1,9 @@ +*.swp +__pycache__ +.pytest_cache +.venv +*.egg-info + +# CDK asset staging directory +.cdk.staging +cdk.out diff --git a/cloud_templates/aws_cdk/OpenSearchPattern/README.md b/cloud_templates/aws_cdk/OpenSearchPattern/README.md new file mode 100644 index 0000000..39b7210 --- /dev/null +++ b/cloud_templates/aws_cdk/OpenSearchPattern/README.md @@ -0,0 +1,154 @@ + +# Welcome to your CDK project! +# IoT Data visulaization with Amazon Opensearch Service + +The `cdk.json` file tells the CDK Toolkit how to execute your app. + +This project is set up like a standard Python project. The initialization +process also creates a virtualenv within this project, stored under the `.venv` +directory. To create the virtualenv it assumes that there is a `python3` +(or `python` for Windows) executable in your path with access to the `venv` +package. If for any reason the automatic creation of the virtualenv fails, +you can create the virtualenv manually. + +To manually create a virtualenv on MacOS and Linux: + +``` +$ python3 -m venv .venv +``` + +After the init process completes and the virtualenv is created, you can use the following +step to activate your virtualenv. + +``` +$ source .venv/bin/activate +``` + +If you are a Windows platform, you would activate the virtualenv like this: + +``` +% .venv\Scripts\activate.bat +``` + +Once the virtualenv is activated, you can install the required dependencies. + +``` +$ pip install -r requirements.txt +``` + +At this point you can now synthesize the CloudFormation template for this code. + +``` +$ cdk synth +``` + +To add additional dependencies, for example other CDK libraries, just add +them to your `setup.py` file and rerun the `pip install -r requirements.txt` +command. + +## Useful commands + + * `cdk ls` list all stacks in the app + * `cdk synth` emits the synthesized CloudFormation template + * `cdk deploy` deploy this stack to your default AWS account/region + * `cdk diff` compare deployed stack with current state + * `cdk docs` open CDK documentation + + ## Context parameters +There are multiple context parameters that you need to set before synthesizing or delpoying this CDK stack. You can specify a context variable either as part of an AWS CDK CLI command, or in `cdk.json`. +To create a command line context variable, use the __--context (-c) option__, as shown in the following example. + +``` +$ cdk cdk synth -c bucket_name=mybucket +``` + +To specify the same context variable and value in the cdk.json file, use the following code. + +``` +{ + "context": { + "bucket_name": "mybucket" + } +} +``` + +In this project, these are the following parameters to be set: + +* `topic_sql` +It is required for IoT Core rule creation to add a simplified SQL syntax to filter messages received on an MQTT topic and push the data elsewhere. +
__Format__: Enter an SQL statement using the following: ```SELECT FROM WHERE ```. For example: ```SELECT temperature FROM 'iot/topic' WHERE temperature > 50```. To learn more, see AWS IoT SQL Reference. + +* `opensearch_domain_name`    `` +The name of the Opensearch domain that will be created. +
__Format__: The name must start with a lowercase letter and must be between 3 and 28 characters. Valid characters are a-z (lowercase only), 0-9, and - (hyphen). + +* `opensearch_index_name`    `` +Before you can search data, you must index it. Indexing is the method by which search engines organize data for fast retrieval. The resulting structure is called, fittingly, an index. The name of the index that the IoT Core will use to send data to Opensearch is set via this parameter. +
__Format__: OpenSearch Service indexes have the following naming restrictions: + + * All letters must be lowercase. + + * Index names cannot begin with `_` or `-`. + + * Index names can't contain `spaces, commas, :, ", *, +, /, \, |, ?, #, >, or <`. + +* `opensearch_type_name`    `` +It resresents the type of the document that is going to be put under the index. Take the following example:
``` { + "_index" : "movies", + "_type" : "_doc", + "_id" : "1", + "_score" : 0.2876821, + "_source" : { + "director" : "Burton, Tim", + "genre" : [ + "Comedy", + "Sci-Fi" + ], + "year" : 1996, + "actor" : [ + "Jack Nicholson", + "Pierce Brosnan", + "Sarah Jessica Parker" + ], + "title" : "Mars Attacks!" + }``` + +* `cognito_user_pool_name`    `` +During the user pool creation process, you must specify a user pool name. This name can't be changed after the user pool has been created. +
__Format__: User pool names must be between one and 128 characters long. They can contain uppercase and lowercase letters (a-z, A-Z), numbers (0-9), and the following special characters: + = , . @ and -. + +* `cognito_user_pool_domain_name`    `` +The domain name for the domain that hosts the sign-up and sign-in pages for your application. +
__Format__: This string can include only lowercase letters, numbers, and hyphens. Don't use a hyphen for the first or last character. Use periods to separate subdomain names. It must be between 1 and 63 characters. + +* `cognito_identity_pool_name`    `` +Identity pools are used to store end user identities. To declare a new identity pool, you should provide a unique name. +
__Format__: It must be between 1 and 128 characters and follow such pattern: ```[\w\s+=,.@-]+ ``` + +* `cognito_user_username`    `` + Each user has a username attribute. Amazon Cognito automatically generates a user name for federated users. You must provide a username attribute to create a native user in the Amazon Cognito directory. After you create a user, you can't change the value of the username attribute. + + * `iot_to_opensearch_rule_name`    `` +The name of the IoT Core rule that is going to be created. +
__Format__: Should be an alphanumeric string that can also contain underscore (_) characters, but no spaces + + * `iot_to_opensearch_role_name`    `` +An IAM role should be created to grant AWS IoT access to Opensearch. This parameter is for setting the name of this role. +
__Format__: Enter a unique role name that contains alphanumeric characters, hyphens, and underscores. A role name can't contain any spaces. + + * `opensearch_domain_capacity_config`    `` +Configures the capacity of the cluster such as the instance type and the number of instances. +
__Format__: Your input must be in the following format: +
+ +``` +# Do not include "opensearch_domain_master_nodes" and "opensearch_domain_warm_nodes" keys if you do not want them in your cluster's config. +{ + "opensearch_domain_data_nodes": , + "opensearch_domain_data_node_instance_type": , + "opensearch_domain_master_nodes": , + "opensearch_domain_warm_nodes": +} +``` + +Enjoy! diff --git a/cloud_templates/aws_cdk/OpenSearchPattern/app.py b/cloud_templates/aws_cdk/OpenSearchPattern/app.py new file mode 100644 index 0000000..1b7a79f --- /dev/null +++ b/cloud_templates/aws_cdk/OpenSearchPattern/app.py @@ -0,0 +1,27 @@ +import os + +import aws_cdk as cdk + +from open_search_pattern.open_search_pattern_stack import OpenSearchPatternStack + + +app = cdk.App() +OpenSearchPatternStack(app, "OpenSearchPatternStack", + # If you don't specify 'env', this stack will be environment-agnostic. + # Account/Region-dependent features and context lookups will not work, + # but a single synthesized template can be deployed anywhere. + + # Uncomment the next line to specialize this stack for the AWS Account + # and Region that are implied by the current CLI configuration. + + #env=cdk.Environment(account=os.getenv('CDK_DEFAULT_ACCOUNT'), region=os.getenv('CDK_DEFAULT_REGION')), + + # Uncomment the next line if you know exactly what Account and Region you + # want to deploy the stack to. */ + + #env=cdk.Environment(account='123456789012', region='us-east-1'), + + # For more information, see https://docs.aws.amazon.com/cdk/latest/guide/environments.html + ) + +app.synth() \ No newline at end of file diff --git a/cloud_templates/aws_cdk/OpenSearchPattern/cdk.json b/cloud_templates/aws_cdk/OpenSearchPattern/cdk.json new file mode 100644 index 0000000..29e85ad --- /dev/null +++ b/cloud_templates/aws_cdk/OpenSearchPattern/cdk.json @@ -0,0 +1,52 @@ +{ + "app": "python3 app.py", + "watch": { + "include": [ + "**" + ], + "exclude": [ + "README.md", + "cdk*.json", + "requirements*.txt", + "source.bat", + "**/__init__.py", + "python/__pycache__", + "tests" + ] + }, + "context": { + "@aws-cdk/aws-apigateway:usagePlanKeyOrderInsensitiveId": true, + "@aws-cdk/core:stackRelativeExports": true, + "@aws-cdk/aws-rds:lowercaseDbIdentifier": true, + "@aws-cdk/aws-lambda:recognizeVersionProps": true, + "@aws-cdk/aws-lambda:recognizeLayerVersion": true, + "@aws-cdk/aws-cloudfront:defaultSecurityPolicyTLSv1.2_2021": true, + "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true, + "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true, + "@aws-cdk/core:checkSecretUsage": true, + "@aws-cdk/aws-iam:minimizePolicies": true, + "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true, + "@aws-cdk/core:validateSnapshotRemovalPolicy": true, + "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true, + "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true, + "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true, + "@aws-cdk/core:target-partitions": [ + "aws", + "aws-cn" + ], + "topic_sql": "SELECT *, parse_time(\"YYYY-MM-dd'T'hh:mm:ss\", timestamp()) as Time FROM 'Opensearch_demo'", + "opensearch_domain_name": "opensearch-demo-domain", + "opensearch_index_name": "iot", + "opensearch_type_name": "_doc", + "iot_to_opensearch_rule_name": "demo_to_opensearch_rule", + "iot_to_opensearch_role_name": "demo_iot_opensearch_role", + "cognito_user_pool_name" : "DemoUserPool", + "cognito_identity_pool_name": "DemoIdentityPool", + "cognito_user_pool_domain_name": "iot-demo-domain", + "cognito_user_username": "admin", + "opensearch_domain_capacity_config": { + "opensearch_domain_data_nodes": 3, + "opensearch_domain_data_node_instance_type": "t3.small.search" + } + } +} diff --git a/cloud_templates/aws_cdk/OpenSearchPattern/open_search_pattern/__init__.py b/cloud_templates/aws_cdk/OpenSearchPattern/open_search_pattern/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cloud_templates/aws_cdk/OpenSearchPattern/open_search_pattern/open_search_pattern_stack.py b/cloud_templates/aws_cdk/OpenSearchPattern/open_search_pattern/open_search_pattern_stack.py new file mode 100644 index 0000000..1e9ef58 --- /dev/null +++ b/cloud_templates/aws_cdk/OpenSearchPattern/open_search_pattern/open_search_pattern_stack.py @@ -0,0 +1,369 @@ +from importlib import resources +from attr import attr +from aws_cdk import ( + Stack, + aws_iam as iam, + aws_iot as iot, + aws_logs as logs, + aws_opensearchservice as opensearch, + aws_cognito as cognito +) +from constructs import Construct +import aws_cdk as cdk +import datetime +import string +import sys +import secrets +import re + +sys.path.append('../') +from common.inputValidation import * +from common.customExceptions import * + +class OpenSearchPatternStack(Stack): + + # Defining class variables + topic_sql = "" + opensearch_domain_name = "" + opensearch_index_name = "" + opensearch_type_name = "" + cognito_user_pool_name = "" + cognito_user_pool_domain_name = "" + cognito_identity_pool_name = "" + cognito_user_username = "" + iot_to_opensearch_rule_name = "" + iot_to_opensearch_role_name = "" + # Default values for OpenSearch domain CapacityConfig + opensearch_domain_data_nodes = 3 + opensearch_domain_data_node_instance_type = "t3.small.search" + opensearch_domain_master_nodes = "" + opensearch_domain_warm_nodes = "" + + def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: + super().__init__(scope, construct_id, **kwargs) + + # Get the context parameters + + # Required parameters for users to set in the CLI command or cdk.json + self.topic_sql = self.node.try_get_context("topic_sql") + + # Optional parameters for users to set in the CLI command or cdk.json + self.opensearch_domain_name = self.node.try_get_context("opensearch_domain_name") + self.opensearch_index_name = self.node.try_get_context("opensearch_index_name") + self.opensearch_type_name = self.node.try_get_context("opensearch_type_name") + self.cognito_user_pool_name = self.node.try_get_context("cognito_user_pool_name") + self.cognito_user_pool_domain_name = self.node.try_get_context("cognito_user_pool_domain_name") + self.cognito_identity_pool_name = self.node.try_get_context("cognito_identity_pool_name") + self.cognito_user_username = self.node.try_get_context("cognito_user_username") + self.iot_to_opensearch_rule_name = self.node.try_get_context("iot_to_opensearch_rule_name") + self.master_user_role_name = self.node.try_get_context("master_user_role_name") + + # Input Validation + self.performInputValidation() + + # Creating cognito identity pool + identityPool = cognito.CfnIdentityPool(self, self.cognito_identity_pool_name, allow_unauthenticated_identities=False) + identityPool.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) + + # Creating an IAM role as the master user for the fine-grained access control + masterUserRole = iam.CfnRole(self, self.master_user_role_name, assume_role_policy_document={ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": "cognito-identity.amazonaws.com" + }, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "cognito-identity.amazonaws.com:aud": identityPool.ref + }, + "ForAnyValue:StringLike": { + "cognito-identity.amazonaws.com:amr": "authenticated" + } + } + }, + { + "Effect": "Allow", + "Principal": { + "Service": "iot.amazonaws.com" + }, + "Action": "sts:AssumeRole" + } + ] + }) + masterUserRole.node.add_dependency(identityPool) + masterUserRole.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) + + # Creating cognito User pool and a domain for it + userPool = cognito.UserPool(self, self.cognito_user_pool_name) + userPool.add_domain(id=self.cognito_user_pool_domain_name, cognito_domain=cognito.CognitoDomainOptions( + domain_prefix=self.cognito_user_pool_domain_name)) + userPool.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) + + # Creating a UserPoolUser with a temporary password + masterUser = cdk.custom_resources.AwsCustomResource(self, "UserPoolUserCreation", + policy=cdk.custom_resources.AwsCustomResourcePolicy.from_statements([ + iam.PolicyStatement(effect=iam.Effect.ALLOW, actions=["cognito-idp:*"], resources=["*"]) + ]), + on_create=cdk.custom_resources.AwsSdkCall( + service='CognitoIdentityServiceProvider', + action='adminCreateUser', + parameters={ + "UserPoolId": userPool.user_pool_id, + "Username": self.cognito_user_username, + "TemporaryPassword": self.randomTemporaryPasswordGenerator(10) + }, + physical_resource_id=cdk.custom_resources.PhysicalResourceId.of('userpoolcreateid' + datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) + )) + + + # Creating a UserPoolGroup in the UserPool + user_pool_group = cognito.CfnUserPoolGroup(self, "master-user-group", group_name="master-user-group", + role_arn=masterUserRole.attr_arn, user_pool_id=userPool.user_pool_id) + user_pool_group.node.add_dependency(userPool) + user_pool_group.node.add_dependency(masterUserRole) + user_pool_group.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) + + # Attaching the user to the group + user_to_group_attachment = cognito.CfnUserPoolUserToGroupAttachment(self, "CDK_master_user_group_attachement", + group_name=user_pool_group.group_name, username=self.cognito_user_username, user_pool_id=userPool.user_pool_id) + user_to_group_attachment.node.add_dependency(user_pool_group) + user_to_group_attachment.node.add_dependency(userPool) + user_to_group_attachment.node.add_dependency(masterUser) + user_to_group_attachment.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) + + # Creating an IAM role to grant Opensearch access to Cognito + cognitoAccessForOpenSearchRole = iam.Role(self, "CDKCognitoAccessForOpenSearch", assumed_by=iam.ServicePrincipal("es.amazonaws.com")) + cognitoAccessForOpenSearchRole.add_managed_policy(iam.ManagedPolicy.from_aws_managed_policy_name("AmazonOpenSearchServiceCognitoAccess")) + cognitoAccessForOpenSearchRole.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) + + # Creating an Opensearch domain + domain = opensearch.Domain(self, self.opensearch_domain_name, + domain_name=self.opensearch_domain_name, + version=opensearch.EngineVersion.OPENSEARCH_1_2, + node_to_node_encryption=True, + encryption_at_rest=opensearch.EncryptionAtRestOptions( + enabled=True + ), + capacity=opensearch.CapacityConfig( + data_nodes=self.opensearch_domain_data_nodes, + data_node_instance_type=self.opensearch_domain_data_node_instance_type + ), + enforce_https=True, + enable_version_upgrade=True, + fine_grained_access_control=opensearch.AdvancedSecurityOptions( + master_user_arn=masterUserRole.attr_arn + ), + cognito_dashboards_auth=opensearch.CognitoOptions( + identity_pool_id=identityPool.ref, + user_pool_id=userPool.user_pool_id, + role=cognitoAccessForOpenSearchRole + )) + domain.add_access_policies(iam.PolicyStatement(effect=iam.Effect.ALLOW, resources=[domain.domain_arn+"/*"], actions=["es:ESHttp*"], principals=[iam.AnyPrincipal()])) + domain.node.add_dependency(identityPool) + domain.node.add_dependency(userPool) + domain.node.add_dependency(cognitoAccessForOpenSearchRole) + domain.node.add_dependency(masterUserRole) + domain.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) + + # Getting the Client ID of the opensearch client in UserPool + userPoolClients = cdk.custom_resources.AwsCustomResource(self, "ClientIdResource", + policy=cdk.custom_resources.AwsCustomResourcePolicy.from_sdk_calls( + resources=[userPool.user_pool_arn] + ), + on_create=cdk.custom_resources.AwsSdkCall( + service='CognitoIdentityServiceProvider', + action='listUserPoolClients', + parameters={"UserPoolId" : userPool.user_pool_id}, + physical_resource_id=cdk.custom_resources.PhysicalResourceId.of(f'ClientId-{self.cognito_user_pool_domain_name}') + )) + userPoolClients.node.add_dependency(domain) + clientID = userPoolClients.get_response_field('UserPoolClients.0.ClientId') + + # Changing Use default role to Choose role from token in Authentication Providers and For Role resolution, choose DENY. (Identity Pool Settigns) + roleAttachement = cognito.CfnIdentityPoolRoleAttachment(self, "CDKIdentityPoolAttachment", + identity_pool_id=identityPool.ref, + roles={}, + role_mappings={ + "role_mappings_key": cognito.CfnIdentityPoolRoleAttachment.RoleMappingProperty( + type='Token', + ambiguous_role_resolution='Deny', + identity_provider=f'cognito-idp.{self.region}.amazonaws.com/{userPool.user_pool_id}:{clientID}' + ) + }) + roleAttachement.node.add_dependency(domain) + roleAttachement.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) + + # Changing masterUserRole to also grant IoT access to Opensearch + masterUserRole.policies = [iam.CfnRole.PolicyProperty( + policy_document={ + "Version": "2012-10-17", + "Statement": [ + { + "Action": "iotanalytics:BatchPutMessage", + "Resource": f"arn:aws:iotanalytics:{self.region}:{self.account}:domain/{self.opensearch_domain_name}/*", + "Effect": "Allow" + } + ] + }, + policy_name="master_useriot_integration")] + + # Creating a cloud watch log group to capture any errors while sending the data through the IoT rule + log_group = logs.LogGroup(self, "iot_to_opensearch_log_group" , log_group_name="iot_to_opensearch_log_group", removal_policy=cdk.RemovalPolicy.DESTROY) + + # Creating the role to access the cloudwatch log group from IoT Core + iot_cloudwatch_log_role = iam.Role(self, "iot_cloudwatch_error_log_role", assumed_by=iam.ServicePrincipal("iot.amazonaws.com")) + iot_cloudwatch_log_role.add_to_policy(iam.PolicyStatement(effect=iam.Effect.ALLOW, resources=[log_group.log_group_arn], actions=["logs:CreateLogStream","logs:DescribeLogStreams","logs:PutLogEvents"])) + iot_cloudwatch_log_role.node.add_dependency(log_group) + iot_cloudwatch_log_role.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) + + # Creating the IoT Core Rule + Setting the error action to log in the cloud watch log group + topic_rule = iot.CfnTopicRule(self, self.iot_to_opensearch_rule_name, topic_rule_payload=iot.CfnTopicRule.TopicRulePayloadProperty( + actions=[iot.CfnTopicRule.ActionProperty( open_search=iot.CfnTopicRule.OpenSearchActionProperty( + endpoint= "https://" + domain.domain_endpoint, + id="${newuuid()}", + index=self.opensearch_index_name, + role_arn=masterUserRole.attr_arn, + type=self.opensearch_type_name + ) + )], sql=self.topic_sql, + aws_iot_sql_version = '2016-03-23', + error_action = iot.CfnTopicRule.ActionProperty( + cloudwatch_logs=iot.CfnTopicRule.CloudwatchLogsActionProperty( + log_group_name=log_group.log_group_name, + role_arn=iot_cloudwatch_log_role.role_arn + )) + )) + topic_rule.node.add_dependency(domain) + topic_rule.node.add_dependency(masterUserRole) + topic_rule.node.add_dependency(log_group) + topic_rule.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) + + def randomTemporaryPasswordGenerator(self, length): + + source = string.ascii_letters + string.digits + string.punctuation + password = secrets.choice(string.ascii_lowercase) + password += secrets.choice(string.ascii_uppercase) + password += secrets.choice(string.digits) + password += secrets.choice(string.punctuation) + + for i in range(length-4): + password += secrets.choice(source) + + char_list = list(password) + secrets.SystemRandom().shuffle(char_list) + password = ''.join(char_list) + + print(f"The temporary password created for the Cognito user is {password}\nPlease keep this password for the first login and then change it to a secure one of your own.") + return password + + def performInputValidation(self): + self.validateTopicSQL(self.topic_sql) + self.validateOpensearchDomainName(self.opensearch_domain_name) + self.validateOpensearchIndexName(self.opensearch_index_name) + self.validateOpensearchTypeName(self.opensearch_type_name) + self.validateCognitoUserPoolName(self.cognito_user_pool_name) + self.validateCognitoUserPoolDomainName(self.cognito_user_pool_domain_name) + self.validateCognitoIdentityPoolName(self.cognito_identity_pool_name) + self.validateCognitoUserUsername(self.cognito_user_username) + self.validateIoTRuleName(self.iot_to_opensearch_rule_name) + self.validateIAMRoleName(self.master_user_role_name) + self.validateCapacityConfig(self.node.try_get_context("opensearch_domain_capacity_config")) + + def validateTopicSQL(self, input): + if not input: + raise NoSQL + elif type(input) != str: + raise WrongFormattedInput("The input sql statement does not have a right format. Please refer to README.md for more information.") + return + + def validateOpensearchDomainName(self, input): + if not input: + self.opensearch_domain_name = "opensearch-demo-domain" + elif type(input) != str: + raise WrongFormattedInput("The Opensearch domain name should be of a string format.") + elif not input[0].islower(): + raise WrongFormattedInput("The Opensearch domain name must start with a lowercase letter.") + else: + checkInputLength(self, 3, 28, input, "OpenSearch domain") + checkInputPattern(self, r'^[a-z0-9-]+$', input, "OpenSearch domain") + + def validateOpensearchIndexName(self, input): + not_allowed_characters = [' ', ',', ':', '\"', '*', '+', '/', '\\', '|', '?', '#', '>' , '<'] + if not input: + self.opensearch_index_name = "iot" + elif type(input) != str: + raise WrongFormattedInput("The Opensearch index name should be of a string format.") + elif not input.islower(): + raise WrongFormattedInput("All letters of the Opensearch index name must be lowercase.") + elif input[0] == "_" or input[0] == "-": + raise WrongFormattedInput("Opensearch index names cannot begin with `_` or `-`.") + elif any(c in input for c in not_allowed_characters): + raise WrongFormattedInput("Opensearch index names cannot contain `spaces, commas, :, \", *, +, /, \\, |, ?, #, >, or <`.") + else: + return + + def validateOpensearchTypeName(self, input): + if not input: + self.opensearch_type_name = "_doc" + else: + return + + def validateCognitoUserPoolName(self, input): + if not input: + self.cognito_user_pool_name = "DemoUserPool" + elif type(input) != str: + raise WrongFormattedInput("The Cognito UserPool name should be of a string format.") + else: + checkInputLength(self, 1, 128, input, "Cognito UserPool") + checkInputPattern(self, r'^[a-zA-Z0-9-+=,@\.]+$', input, "Cognito UserPool") + + def validateCognitoUserPoolDomainName(self, input): + if not input: + self.cognito_user_pool_domain_name = "iot-demo-domain" + elif type(input) != str: + raise WrongFormattedInput("The Cognito UserPool domain name should be of a string format.") + else: + checkInputLength(self, 1, 63, input, "Cognito UserPool domain") + checkInputPattern(self, r'^[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?$', input, "Cognito UserPool domain") + + def validateCognitoIdentityPoolName(self, input): + if not input: + self.cognito_identity_pool_name = "DemoIdentityPool" + elif type(input) != str: + raise WrongFormattedInput("The Cognito IdentityPool name should be of a string format.") + else: + checkInputLength(self, 1, 128, input, "Cognito IdentityPool") + checkInputPattern(self, r'^[\w\s+=,.@-]+$', input, "Cognito IdentityPool") + + def validateCognitoUserUsername(self, input): + if not input: + self.cognito_user_username = "admin" + else: + return + + def validateIoTRuleName(self, input): + if not input: + self.iot_to_opensearch_rule_name = "demo_iot_opensearch_rule" + elif type(input) != str: + raise WrongFormattedInput("The provided input for IoT topic rule name is not of type string.") + else: + checkInputPattern(self, r'^[a-zA-Z0-9-_\.]+$', input, "IoT rule") + + def validateIAMRoleName(self, input): + if not input: + self.master_user_role_name = "demo_iot_opensearch_role" + elif type(input) != str: + raise WrongFormattedInput("The provided input for the IAM role name is not of type string.") + else: + checkInputLength(self, 1, 64, input, "IAM role") + checkInputPattern(self, r'^[a-zA-Z0-9+=,@-_\.]+$', input, "IAM role") + + def validateCapacityConfig(self, input): + if input: + self.opensearch_domain_data_nodes = input["opensearch_domain_data_nodes"] if "opensearch_domain_data_nodes" in input else 3 + self.opensearch_domain_data_node_instance_type = input["opensearch_domain_data_node_instance_type"] if "opensearch_domain_data_node_instance_type" in input else "t3.small.search" + self.opensearch_domain_master_nodes = input["opensearch_domain_master_nodes"] if "opensearch_domain_master_nodes" in input else "" + self.opensearch_domain_warm_nodes = input["opensearch_domain_warm_nodes"] if "opensearch_domain_warm_nodes" in input else "" \ No newline at end of file diff --git a/cloud_templates/aws_cdk/OpenSearchPattern/requirements-dev.txt b/cloud_templates/aws_cdk/OpenSearchPattern/requirements-dev.txt new file mode 100644 index 0000000..9270945 --- /dev/null +++ b/cloud_templates/aws_cdk/OpenSearchPattern/requirements-dev.txt @@ -0,0 +1 @@ +pytest==6.2.5 diff --git a/cloud_templates/aws_cdk/OpenSearchPattern/requirements.txt b/cloud_templates/aws_cdk/OpenSearchPattern/requirements.txt new file mode 100644 index 0000000..0822bbe --- /dev/null +++ b/cloud_templates/aws_cdk/OpenSearchPattern/requirements.txt @@ -0,0 +1,2 @@ +aws-cdk-lib==2.37.1 +constructs>=10.0.0,<11.0.0 diff --git a/cloud_templates/aws_cdk/OpenSearchPattern/source.bat b/cloud_templates/aws_cdk/OpenSearchPattern/source.bat new file mode 100644 index 0000000..9e1a834 --- /dev/null +++ b/cloud_templates/aws_cdk/OpenSearchPattern/source.bat @@ -0,0 +1,13 @@ +@echo off + +rem The sole purpose of this script is to make the command +rem +rem source .venv/bin/activate +rem +rem (which activates a Python virtualenv on Linux or Mac OS X) work on Windows. +rem On Windows, this command just runs this batch file (the argument is ignored). +rem +rem Now we don't need to document a Windows command for activating a virtualenv. + +echo Executing .venv\Scripts\activate.bat for you +.venv\Scripts\activate.bat diff --git a/cloud_templates/aws_cdk/OpenSearchPattern/tests/__init__.py b/cloud_templates/aws_cdk/OpenSearchPattern/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cloud_templates/aws_cdk/OpenSearchPattern/tests/unit/__init__.py b/cloud_templates/aws_cdk/OpenSearchPattern/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cloud_templates/aws_cdk/OpenSearchPattern/tests/unit/test_open_search_pattern_stack.py b/cloud_templates/aws_cdk/OpenSearchPattern/tests/unit/test_open_search_pattern_stack.py new file mode 100644 index 0000000..c1b9832 --- /dev/null +++ b/cloud_templates/aws_cdk/OpenSearchPattern/tests/unit/test_open_search_pattern_stack.py @@ -0,0 +1,798 @@ +import aws_cdk as core +import aws_cdk.assertions as assertions +from aws_cdk.assertions import Match +import pytest + +from open_search_pattern.open_search_pattern_stack import OpenSearchPatternStack + +# Setting the context for the app +app = core.App(context={ + "topic_sql": "SELECT * FROM 'Opensearch_demo'", + "opensearch_domain_name": "cdk-opensearch-domain", + "opensearch_index_name": "measurements", + "opensearch_type_name": "_doc", + "iot_to_opensearch_rule_name": "cdk_iot_opensearchFineGrained_rule", + "master_user_role_name": "masterUserRole", + "cognito_user_pool_name" : "CDKUserPool", + "cognito_identity_pool_name": "CDKIdentityPool", + "cognito_user_pool_domain_name": "cdk-iot-domain", + "cognito_user_username": "admin" +}) + +stack = OpenSearchPatternStack(app, "open-search-pattern") +template = assertions.Template.from_stack(stack) + +# Defining Capture objects for obtaining values in tests +userPool_ref_capture = assertions.Capture() +userpooluser_lambda_ref_capture = assertions.Capture() +userPool_group_name = assertions.Capture() +userPool_group_ref_capture = assertions.Capture() +userPool_domain_ref_capture = assertions.Capture() +fineGrainedRole_ref_capture = assertions.Capture() +identity_pool_ref_capture = assertions.Capture() +cognitoAccessForOpenSearch_role_ref_capture = assertions.Capture() +opensearch_domain_ref_capture = assertions.Capture() +opensearch_domain_access_policy_ref_capture = assertions.Capture() +custom_clientIdResource_ref_capture = assertions.Capture() +log_group_ref_capture = assertions.Capture() +log_group_role_ref_capture = assertions.Capture() +log_group_policy_ref_capture = assertions.Capture() +userPoolUserCreation_ref_capture = assertions.Capture() +userPoolUserCreationCustomResourcePolicy_ref_capture = assertions.Capture() +masterUserRole_ref_capture = assertions.Capture() + +# Testing the resources' creation and properties + +def test_identity_pool_creation(): + template.has_resource("AWS::Cognito::IdentityPool", {"DeletionPolicy":"Delete", "UpdateReplacePolicy":"Delete"}) + template.resource_count_is("AWS::Cognito::IdentityPool",1) + +def test_identity_pool_properties(): + template.has_resource_properties("AWS::Cognito::IdentityPool", { + "AllowUnauthenticatedIdentities": False + }) + +def test_masterUser_IAM_role_properties(): + template.has_resource_properties("AWS::IAM::Role", { + "AssumeRolePolicyDocument": { + "Version": Match.any_value(), + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": "cognito-identity.amazonaws.com" + }, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "cognito-identity.amazonaws.com:aud": { + "Ref": app.node.try_get_context("cognito_identity_pool_name") + } + }, + "ForAnyValue:StringLike": { + "cognito-identity.amazonaws.com:amr": "authenticated" + } + } + }, + { + "Effect": "Allow", + "Principal": { + "Service": "iot.amazonaws.com" + }, + "Action": "sts:AssumeRole" + } + ] + }, + "Policies": [{ + "PolicyDocument": { + "Version": Match.any_value(), + "Statement": [ + { + "Action": "iotanalytics:BatchPutMessage", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:aws:iotanalytics:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + f':domain/{app.node.try_get_context("opensearch_domain_name")}/*' + ] + ] + }, + "Effect": "Allow" + } + ] + }, + "PolicyName": Match.any_value() + }] + }) + +def test_user_pool_creation(): + template.has_resource("AWS::Cognito::UserPool", {"DeletionPolicy":"Delete", "UpdateReplacePolicy":"Delete"}) + template.resource_count_is("AWS::Cognito::UserPool",1) + +def test_user_pool_properties(): + template.has_resource_properties("AWS::Cognito::UserPool", { + "AccountRecoverySetting": { + "RecoveryMechanisms": [ + { + "Name": "verified_phone_number", + "Priority": 1 + }, + { + "Name": "verified_email", + "Priority": 2 + } + ] + }, + "AdminCreateUserConfig": { + "AllowAdminCreateUserOnly": True + } + }) + +def test_user_pool_domain_creation(): + template.has_resource("AWS::Cognito::UserPoolDomain", {}) + template.resource_count_is("AWS::Cognito::UserPoolDomain",1) + +def test_user_pool_domain_properties(): + template.has_resource_properties("AWS::Cognito::UserPoolDomain", { + "Domain": app.node.try_get_context("cognito_user_pool_domain_name"), + "UserPoolId": { + "Ref": userPool_ref_capture + } + }) + +def test_user_pool_user_creation(): + template.has_resource("Custom::AWS", {"DeletionPolicy":"Delete", "UpdateReplacePolicy":"Delete"}) + +def test_user_pool_user_properties(): + template.has_resource_properties("Custom::AWS", { + "ServiceToken": { + "Fn::GetAtt": [ + userpooluser_lambda_ref_capture, + "Arn" + ] + }, + "Create": Match.any_value(), + "InstallLatestAwsSdk": True + }) + +def test_user_pool_user_group_creation(): + template.has_resource("AWS::Cognito::UserPoolGroup", {"DeletionPolicy":"Delete", "UpdateReplacePolicy":"Delete"}) + template.resource_count_is("AWS::Cognito::UserPoolGroup", 1) + +def test_user_pool_user_group_properties(): + template.has_resource_properties("AWS::Cognito::UserPoolGroup", { + "UserPoolId": { + "Ref": userPool_ref_capture.as_string() + }, + "GroupName": userPool_group_name, + "RoleArn": { + "Fn::GetAtt": [ + app.node.try_get_context("master_user_role_name"), + "Arn" + ] + } + }) + +def test_userPoolUserToGroupAttachment_creation(): + template.has_resource("AWS::Cognito::UserPoolUserToGroupAttachment", {"DeletionPolicy":"Delete", "UpdateReplacePolicy":"Delete"}) + template.resource_count_is("AWS::Cognito::UserPoolUserToGroupAttachment", 1) + +def test_userPoolUserToGroupAttachment_properties(): + template.has_resource_properties("AWS::Cognito::UserPoolUserToGroupAttachment", { + "GroupName": userPool_group_name.as_string(), + "Username": app.node.try_get_context("cognito_user_username"), + "UserPoolId": { + "Ref": userPool_ref_capture.as_string() + } + }) + +def test_CognitoAccessForOpenSearch_role_properties(): + template.has_resource_properties("AWS::IAM::Role", { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "es.amazonaws.com" + } + }], + "Version": Match.any_value() + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/AmazonOpenSearchServiceCognitoAccess" + ]] + }] + }) + +def test_opensearch_domain_creation(): + template.has_resource("AWS::OpenSearchService::Domain", {"DeletionPolicy":"Delete", "UpdateReplacePolicy":"Delete"}) + template.resource_count_is("AWS::OpenSearchService::Domain",1) + +def test_opensearch_domain_properties(): + template.has_resource_properties("AWS::OpenSearchService::Domain", { + "AdvancedSecurityOptions": { + "Enabled": True, + "InternalUserDatabaseEnabled": False, + "MasterUserOptions": { + "MasterUserARN": { + "Fn::GetAtt": [ + fineGrainedRole_ref_capture, + "Arn" + ] + } + } + }, + "ClusterConfig": { + "DedicatedMasterEnabled": False, + "InstanceCount": Match.any_value(), + "InstanceType": Match.any_value(), + "ZoneAwarenessEnabled": False + }, + "CognitoOptions": { + "Enabled": True, + "IdentityPoolId": { + "Ref": identity_pool_ref_capture + }, + "RoleArn": { + "Fn::GetAtt": [ + cognitoAccessForOpenSearch_role_ref_capture, + "Arn" + ] + }, + "UserPoolId": { + "Ref": userPool_ref_capture.as_string() + } + }, + "DomainEndpointOptions": { + "EnforceHTTPS": True, + "TLSSecurityPolicy": Match.any_value() + }, + "DomainName": app.node.try_get_context("opensearch_domain_name"), + "EBSOptions": { + "EBSEnabled": True, + "VolumeSize": Match.any_value(), + "VolumeType": Match.any_value() + }, + "EncryptionAtRestOptions": { + "Enabled": True + }, + "EngineVersion": Match.any_value(), + "LogPublishingOptions": {}, + "NodeToNodeEncryptionOptions": { + "Enabled": True + } + }) + +def test_opensearch_access_policy_properties(): + template.has_resource_properties("AWS::IAM::Policy", { + "PolicyDocument": { + "Statement": [ + { + "Action": "es:UpdateDomainConfig", + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + opensearch_domain_ref_capture, + "Arn" + ] + } + } + ], + "Version": Match.any_value() + }, + "PolicyName": opensearch_domain_access_policy_ref_capture, + "Roles": [ + { + "Ref": Match.any_value() + } + ] + }) + +def test_IdentityPoolRoleAttachment_creation(): + template.has_resource("AWS::Cognito::IdentityPoolRoleAttachment", {"DeletionPolicy":"Delete", "UpdateReplacePolicy":"Delete"}) + template.resource_count_is("AWS::Cognito::IdentityPoolRoleAttachment", 1) + +def test_IdentityPoolRoleAttachment_properties(): + template.has_resource_properties("AWS::Cognito::IdentityPoolRoleAttachment", { + "IdentityPoolId": { + "Ref": identity_pool_ref_capture.as_string() + }, + "RoleMappings": { + "role_mappings_key": { + "AmbiguousRoleResolution": "Deny", + "IdentityProvider": { + "Fn::Join": [ + "", + [ + "cognito-idp.", + { + "Ref": "AWS::Region" + }, + ".amazonaws.com/", + { + "Ref": userPool_ref_capture.as_string() + }, + ":", + { + "Fn::GetAtt": [ + custom_clientIdResource_ref_capture, + "UserPoolClients.0.ClientId" + ] + } + ]] + }, + "Type": "Token" + } + }, + "Roles": {} + }) + +def test_log_group_creation(): + template.has_resource("AWS::Logs::LogGroup", {"DeletionPolicy":"Delete", "UpdateReplacePolicy":"Delete"}) + template.resource_count_is("AWS::Logs::LogGroup",1) + +def test_log_group_properties(): + template.has_resource_properties("AWS::Logs::LogGroup", { + "LogGroupName": "iot_to_opensearch_log_group", + "RetentionInDays": Match.any_value() + }) + +def test_log_group_role_properties(): + template.has_resource_properties("AWS::IAM::Role", { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "iot.amazonaws.com" + } + } + ], + "Version": Match.any_value() + } + }) + +def test_log_group_policy_properties(): + template.has_resource_properties("AWS::IAM::Policy", { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "logs:CreateLogStream", + "logs:DescribeLogStreams", + "logs:PutLogEvents" + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + log_group_ref_capture, + "Arn" + ] + } + } + ], + "Version": Match.any_value() + }, + "PolicyName": log_group_policy_ref_capture, + "Roles": [ + { + "Ref": log_group_role_ref_capture + } + ] + }) + +def test_iot_rule_creation(): + template.has_resource("AWS::IoT::TopicRule", {"DeletionPolicy":"Delete", "UpdateReplacePolicy":"Delete"}) + template.resource_count_is("AWS::IoT::TopicRule",1) + +def test_iot_rule_properties(): + template.has_resource_properties("AWS::IoT::TopicRule", { + "TopicRulePayload": { + "Actions": [ + { + "OpenSearch": { + "Endpoint": { + "Fn::Join": [ + "", + [ + "https://", + { + "Fn::GetAtt": [ + opensearch_domain_ref_capture.as_string(), + "DomainEndpoint" + ] + } + ] + ] + }, + "Id": "${newuuid()}", + "Index": app.node.try_get_context("opensearch_index_name"), + "RoleArn": { + "Fn::GetAtt": [ + fineGrainedRole_ref_capture.as_string(), + "Arn" + ] + }, + "Type": app.node.try_get_context("opensearch_type_name") + } + } + ], + "ErrorAction": { + "CloudwatchLogs": { + "LogGroupName": { + "Ref": log_group_ref_capture.as_string() + }, + "RoleArn": { + "Fn::GetAtt": [ + log_group_role_ref_capture.as_string(), + "Arn" + ] + } + } + }, + "Sql": app.node.try_get_context("topic_sql") + } + }) + +# Testing dependencies between the resources + +def test_masterUser_role_dependencies(): + template.has_resource("AWS::IAM::Role", { + "DependsOn": [ + identity_pool_ref_capture.as_string() + ] + }) + +def test_userPoolUserToGroupAttachment_dependencies(): + template.has_resource("AWS::Cognito::UserPoolUserToGroupAttachment", { + "DependsOn": [ + userPool_domain_ref_capture, + userPool_ref_capture.as_string(), + userPool_group_ref_capture, + userPoolUserCreationCustomResourcePolicy_ref_capture, + userPoolUserCreation_ref_capture + ] + }) + +def test_opensearch_domain_dependencies(): + template.has_resource("AWS::OpenSearchService::Domain", { + "DependsOn": [ + cognitoAccessForOpenSearch_role_ref_capture.as_string(), + identity_pool_ref_capture.as_string(), + userPool_domain_ref_capture.as_string(), + userPool_ref_capture.as_string(), + masterUserRole_ref_capture + ] + }) + +# Testing input validation process + +def test_no_sql(): + test_app = core.App(context= { + "opensearch_domain_name": "cdk-opensearch-domain", + "opensearch_index_name": "measurements", + "opensearch_type_name": "_doc", + "iot_to_opensearch_rule_name": "cdk_iot_opensearchFineGrained_rule", + "master_user_role_name": "masterUserRole", + "cognito_user_pool_name" : "CDKUserPool", + "cognito_identity_pool_name": "CDKIdentityPool", + "cognito_user_pool_domain_name": "cdk-iot-domain", + "cognito_user_username": "admin" + }) + with pytest.raises(Exception, match=r"No sql statemtnt .*"): + stack = OpenSearchPatternStack(test_app, "open-search-pattern") + template = assertions.Template.from_stack(stack) + +def test_wrong_sql(): + test_app = core.App(context= { + "topic_sql": ["SELECT * FROM 'Opensearch_demo'"], + "opensearch_domain_name": "cdk-opensearch-domain", + "opensearch_index_name": "measurements", + "opensearch_type_name": "_doc", + "iot_to_opensearch_rule_name": "cdk_iot_opensearchFineGrained_rule", + "master_user_role_name": "masterUserRole", + "cognito_user_pool_name" : "CDKUserPool", + "cognito_identity_pool_name": "CDKIdentityPool", + "cognito_user_pool_domain_name": "cdk-iot-domain", + "cognito_user_username": "admin" + }) + with pytest.raises(Exception, match=r"The input sql statement does not have a right format. .*"): + stack = OpenSearchPatternStack(test_app, "open-search-pattern") + template = assertions.Template.from_stack(stack) + +def test_wrong_opensearch_domain(): + test_app = core.App(context= { + "topic_sql": "SELECT * FROM 'Opensearch_demo'", + "opensearch_domain_name": ["cdk-opensearch-domain"], + "opensearch_index_name": "measurements", + "opensearch_type_name": "_doc", + "iot_to_opensearch_rule_name": "cdk_iot_opensearchFineGrained_rule", + "master_user_role_name": "masterUserRole", + "cognito_user_pool_name" : "CDKUserPool", + "cognito_identity_pool_name": "CDKIdentityPool", + "cognito_user_pool_domain_name": "cdk-iot-domain", + "cognito_user_username": "admin" + }) + with pytest.raises(Exception, match=r"The Opensearch domain name should be of a string format."): + stack = OpenSearchPatternStack(test_app, "open-search-pattern") + template = assertions.Template.from_stack(stack) + + test_app = core.App(context= { + "topic_sql": "SELECT * FROM 'Opensearch_demo'", + "opensearch_domain_name": "x" * 30, + "opensearch_index_name": "measurements", + "opensearch_type_name": "_doc", + "iot_to_opensearch_rule_name": "cdk_iot_opensearchFineGrained_rule", + "master_user_role_name": "masterUserRole", + "cognito_user_pool_name" : "CDKUserPool", + "cognito_identity_pool_name": "CDKIdentityPool", + "cognito_user_pool_domain_name": "cdk-iot-domain", + "cognito_user_username": "admin" + }) + with pytest.raises(Exception, match=r"Invalid input length .*"): + stack = OpenSearchPatternStack(test_app, "open-search-pattern") + template = assertions.Template.from_stack(stack) + + test_app = core.App(context= { + "topic_sql": "SELECT * FROM 'Opensearch_demo'", + "opensearch_domain_name": "Domain", + "opensearch_index_name": "measurements", + "opensearch_type_name": "_doc", + "iot_to_opensearch_rule_name": "cdk_iot_opensearchFineGrained_rule", + "master_user_role_name": "masterUserRole", + "cognito_user_pool_name" : "CDKUserPool", + "cognito_identity_pool_name": "CDKIdentityPool", + "cognito_user_pool_domain_name": "cdk-iot-domain", + "cognito_user_username": "admin" + }) + with pytest.raises(Exception, match=r"The Opensearch domain name must start with a lowercase letter."): + stack = OpenSearchPatternStack(test_app, "open-search-pattern") + template = assertions.Template.from_stack(stack) + + test_app = core.App(context= { + "topic_sql": "SELECT * FROM 'Opensearch_demo'", + "opensearch_domain_name": "domain_iot", + "opensearch_index_name": "measurements", + "opensearch_type_name": "_doc", + "iot_to_opensearch_rule_name": "cdk_iot_opensearchFineGrained_rule", + "master_user_role_name": "masterUserRole", + "cognito_user_pool_name" : "CDKUserPool", + "cognito_identity_pool_name": "CDKIdentityPool", + "cognito_user_pool_domain_name": "cdk-iot-domain", + "cognito_user_username": "admin" + }) + with pytest.raises(Exception, match=r"Invalid input pattern .*"): + stack = OpenSearchPatternStack(test_app, "open-search-pattern") + template = assertions.Template.from_stack(stack) + +def test_wrong_opensearch_index(): + test_app = core.App(context= { + "topic_sql": "SELECT * FROM 'Opensearch_demo'", + "opensearch_domain_name": "cdk-opensearch-domain", + "opensearch_index_name": "Index", + "opensearch_type_name": "_doc", + "iot_to_opensearch_rule_name": "cdk_iot_opensearchFineGrained_rule", + "master_user_role_name": "masterUserRole", + "cognito_user_pool_name" : "CDKUserPool", + "cognito_identity_pool_name": "CDKIdentityPool", + "cognito_user_pool_domain_name": "cdk-iot-domain", + "cognito_user_username": "admin" + }) + with pytest.raises(Exception, match=r"All letters of the Opensearch index name must be lowercase."): + stack = OpenSearchPatternStack(test_app, "open-search-pattern") + template = assertions.Template.from_stack(stack) + + test_app = core.App(context= { + "topic_sql": "SELECT * FROM 'Opensearch_demo'", + "opensearch_domain_name": "cdk-opensearch-domain", + "opensearch_index_name": "_myindex", + "opensearch_type_name": "_doc", + "iot_to_opensearch_rule_name": "cdk_iot_opensearchFineGrained_rule", + "master_user_role_name": "masterUserRole", + "cognito_user_pool_name" : "CDKUserPool", + "cognito_identity_pool_name": "CDKIdentityPool", + "cognito_user_pool_domain_name": "cdk-iot-domain", + "cognito_user_username": "admin" + }) + with pytest.raises(Exception, match=r"Opensearch index names cannot begin with `_` or `-`."): + stack = OpenSearchPatternStack(test_app, "open-search-pattern") + template = assertions.Template.from_stack(stack) + + test_app = core.App(context= { + "topic_sql": "SELECT * FROM 'Opensearch_demo'", + "opensearch_domain_name": "cdk-opensearch-domain", + "opensearch_index_name": "my\\index", + "opensearch_type_name": "_doc", + "iot_to_opensearch_rule_name": "cdk_iot_opensearchFineGrained_rule", + "master_user_role_name": "masterUserRole", + "cognito_user_pool_name" : "CDKUserPool", + "cognito_identity_pool_name": "CDKIdentityPool", + "cognito_user_pool_domain_name": "cdk-iot-domain", + "cognito_user_username": "admin" + }) + with pytest.raises(Exception, match=r"Opensearch index names cannot contain .*"): + stack = OpenSearchPatternStack(test_app, "open-search-pattern") + template = assertions.Template.from_stack(stack) + +def test_wrong_userPool_domain_name(): + test_app = core.App(context= { + "topic_sql": "SELECT * FROM 'Opensearch_demo'", + "opensearch_domain_name": "cdk-opensearch-domain", + "opensearch_index_name": "index", + "opensearch_type_name": "_doc", + "iot_to_opensearch_rule_name": "cdk_iot_opensearchFineGrained_rule", + "master_user_role_name": "masterUserRole", + "cognito_user_pool_name" : "CDKUserPool", + "cognito_identity_pool_name": "CDKIdentityPool", + "cognito_user_pool_domain_name": "X" * 64, + "cognito_user_username": "admin" + }) + with pytest.raises(Exception, match=r"Invalid input length .*"): + stack = OpenSearchPatternStack(test_app, "open-search-pattern") + template = assertions.Template.from_stack(stack) + + test_app = core.App(context= { + "topic_sql": "SELECT * FROM 'Opensearch_demo'", + "opensearch_domain_name": "cdk-opensearch-domain", + "opensearch_index_name": "index", + "opensearch_type_name": "_doc", + "iot_to_opensearch_rule_name": "cdk_iot_opensearchFineGrained_rule", + "master_user_role_name": "masterUserRole", + "cognito_user_pool_name" : "CDKUserPool", + "cognito_identity_pool_name": "CDKIdentityPool", + "cognito_user_pool_domain_name": "-cdk-iot-domain", + "cognito_user_username": "admin" + }) + with pytest.raises(Exception, match=r"Invalid input pattern .*"): + stack = OpenSearchPatternStack(test_app, "open-search-pattern") + template = assertions.Template.from_stack(stack) + + test_app = core.App(context= { + "topic_sql": "SELECT * FROM 'Opensearch_demo'", + "opensearch_domain_name": "cdk-opensearch-domain", + "opensearch_index_name": "index", + "opensearch_type_name": "_doc", + "iot_to_opensearch_rule_name": "cdk_iot_opensearchFineGrained_rule", + "master_user_role_name": "masterUserRole", + "cognito_user_pool_name" : "CDKUserPool", + "cognito_identity_pool_name": "CDKIdentityPool", + "cognito_user_pool_domain_name": "Cdk-iot-domain", + "cognito_user_username": "admin" + }) + with pytest.raises(Exception, match=r"Invalid input pattern .*"): + stack = OpenSearchPatternStack(test_app, "open-search-pattern") + template = assertions.Template.from_stack(stack) + +def test_wrong_userPool_name(): + test_app = core.App(context= { + "topic_sql": "SELECT * FROM 'Opensearch_demo'", + "opensearch_domain_name": "cdk-opensearch-domain", + "opensearch_index_name": "index", + "opensearch_type_name": "_doc", + "iot_to_opensearch_rule_name": "cdk_iot_opensearchFineGrained_rule", + "master_user_role_name": "masterUserRole", + "cognito_user_pool_name" : "c" * 129, + "cognito_identity_pool_name": "CDKIdentityPool", + "cognito_user_pool_domain_name": "cdk-iot-domain", + "cognito_user_username": "admin" + }) + with pytest.raises(Exception, match=r"Invalid input length .*"): + stack = OpenSearchPatternStack(test_app, "open-search-pattern") + template = assertions.Template.from_stack(stack) + + test_app = core.App(context= { + "topic_sql": "SELECT * FROM 'Opensearch_demo'", + "opensearch_domain_name": "cdk-opensearch-domain", + "opensearch_index_name": "index", + "opensearch_type_name": "_doc", + "iot_to_opensearch_rule_name": "cdk_iot_opensearchFineGrained_rule", + "master_user_role_name": "masterUserRole", + "cognito_user_pool_name" : "CDK#UserPool", + "cognito_identity_pool_name": "CDKIdentityPool", + "cognito_user_pool_domain_name": "cdk-iot-domain", + "cognito_user_username": "admin" + }) + with pytest.raises(Exception, match=r"Invalid input pattern .*"): + stack = OpenSearchPatternStack(test_app, "open-search-pattern") + template = assertions.Template.from_stack(stack) + +def test_wrong_identityPool_name(): + test_app = core.App(context= { + "topic_sql": "SELECT * FROM 'Opensearch_demo'", + "opensearch_domain_name": "cdk-opensearch-domain", + "opensearch_index_name": "index", + "opensearch_type_name": "_doc", + "iot_to_opensearch_rule_name": "cdk_iot_opensearchFineGrained_rule", + "master_user_role_name": "masterUserRole", + "cognito_user_pool_name" : "CDKUserPool", + "cognito_identity_pool_name": "c" * 129, + "cognito_user_pool_domain_name": "cdk-iot-domain", + "cognito_user_username": "admin" + }) + with pytest.raises(Exception, match=r"Invalid input length .*"): + stack = OpenSearchPatternStack(test_app, "open-search-pattern") + template = assertions.Template.from_stack(stack) + + test_app = core.App(context= { + "topic_sql": "SELECT * FROM 'Opensearch_demo'", + "opensearch_domain_name": "cdk-opensearch-domain", + "opensearch_index_name": "index", + "opensearch_type_name": "_doc", + "iot_to_opensearch_rule_name": "cdk_iot_opensearchFineGrained_rule", + "master_user_role_name": "masterUserRole", + "cognito_user_pool_name" : "CDKUserPool", + "cognito_identity_pool_name": "CDK#IdentityPool", + "cognito_user_pool_domain_name": "cdk-iot-domain", + "cognito_user_username": "admin" + }) + with pytest.raises(Exception, match=r"Invalid input pattern .*"): + stack = OpenSearchPatternStack(test_app, "open-search-pattern") + template = assertions.Template.from_stack(stack) + +def test_wrong_iot_rule_name(): + test_app = core.App(context= { + "topic_sql": "SELECT * FROM 'Opensearch_demo'", + "opensearch_domain_name": "cdk-opensearch-domain", + "opensearch_index_name": "index", + "opensearch_type_name": "_doc", + "iot_to_opensearch_rule_name": "cdk_@iot_opensearchFineGrained_rule", + "master_user_role_name": "masterUserRole", + "cognito_user_pool_name" : "CDKUserPool", + "cognito_identity_pool_name": "CDKIdentityPool", + "cognito_user_pool_domain_name": "cdk-iot-domain", + "cognito_user_username": "admin" + }) + with pytest.raises(Exception, match=r"Invalid input pattern .*"): + stack = OpenSearchPatternStack(test_app, "open-search-pattern") + template = assertions.Template.from_stack(stack) + +def test_wrong_iam_role_name(): + test_app = core.App(context= { + "topic_sql": "SELECT * FROM 'Opensearch_demo'", + "opensearch_domain_name": "cdk-opensearch-domain", + "opensearch_index_name": "index", + "opensearch_type_name": "_doc", + "iot_to_opensearch_rule_name": "cdk_iot_opensearchFineGrained_rule", + "master_user_role_name": "#masterUserRole", + "cognito_user_pool_name" : "CDKUserPool", + "cognito_identity_pool_name": "CDKIdentityPool", + "cognito_user_pool_domain_name": "cdk-iot-domain", + "cognito_user_username": "admin" + }) + with pytest.raises(Exception, match=r"Invalid input pattern .*"): + stack = OpenSearchPatternStack(test_app, "open-search-pattern") + template = assertions.Template.from_stack(stack) + + test_app = core.App(context= { + "topic_sql": "SELECT * FROM 'Opensearch_demo'", + "opensearch_domain_name": "cdk-opensearch-domain", + "opensearch_index_name": "index", + "opensearch_type_name": "_doc", + "iot_to_opensearch_rule_name": "cdk_iot_opensearchFineGrained_rule", + "master_user_role_name": "c" * 65, + "cognito_user_pool_name" : "CDKUserPool", + "cognito_identity_pool_name": "CDKIdentityPool", + "cognito_user_pool_domain_name": "cdk-iot-domain", + "cognito_user_username": "admin" + }) + with pytest.raises(Exception, match=r"Invalid input length .*"): + stack = OpenSearchPatternStack(test_app, "open-search-pattern") + template = assertions.Template.from_stack(stack) \ No newline at end of file diff --git a/cloud_templates/demo/demo_templates/opensearch_pattern.json b/cloud_templates/demo/demo_templates/opensearch_pattern.json new file mode 100644 index 0000000..18f29c3 --- /dev/null +++ b/cloud_templates/demo/demo_templates/opensearch_pattern.json @@ -0,0 +1,1003 @@ +{ + "Resources": { + "DemoIdentityPool": { + "Type": "AWS::Cognito::IdentityPool", + "Properties": { + "AllowUnauthenticatedIdentities": false + }, + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete", + "Metadata": { + "aws:cdk:path": "OpenSearchPatternStack/DemoIdentityPool" + } + }, + "demoiotopensearchrole": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": "cognito-identity.amazonaws.com" + }, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "cognito-identity.amazonaws.com:aud": { + "Ref": "DemoIdentityPool" + } + }, + "ForAnyValue:StringLike": { + "cognito-identity.amazonaws.com:amr": "authenticated" + } + } + }, + { + "Effect": "Allow", + "Principal": { + "Service": "iot.amazonaws.com" + }, + "Action": "sts:AssumeRole" + } + ] + }, + "Policies": [ + { + "PolicyDocument": { + "Version": "2012-10-17", + "Statement": [ + { + "Action": "iotanalytics:BatchPutMessage", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:aws:iotanalytics:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":domain/opensearch-demo-domain/*" + ] + ] + }, + "Effect": "Allow" + } + ] + }, + "PolicyName": "master_useriot_integration" + } + ] + }, + "DependsOn": [ + "DemoIdentityPool" + ], + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete", + "Metadata": { + "aws:cdk:path": "OpenSearchPatternStack/demo_iot_opensearch_role" + } + }, + "DemoUserPool1AB98549": { + "Type": "AWS::Cognito::UserPool", + "Properties": { + "AccountRecoverySetting": { + "RecoveryMechanisms": [ + { + "Name": "verified_phone_number", + "Priority": 1 + }, + { + "Name": "verified_email", + "Priority": 2 + } + ] + }, + "AdminCreateUserConfig": { + "AllowAdminCreateUserOnly": true + }, + "EmailVerificationMessage": "The verification code to your new account is {####}", + "EmailVerificationSubject": "Verify your new account", + "SmsVerificationMessage": "The verification code to your new account is {####}", + "VerificationMessageTemplate": { + "DefaultEmailOption": "CONFIRM_WITH_CODE", + "EmailMessage": "The verification code to your new account is {####}", + "EmailSubject": "Verify your new account", + "SmsMessage": "The verification code to your new account is {####}" + } + }, + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete", + "Metadata": { + "aws:cdk:path": "OpenSearchPatternStack/DemoUserPool/Resource" + } + }, + "DemoUserPooliotdemodomainEF32B2ED": { + "Type": "AWS::Cognito::UserPoolDomain", + "Properties": { + "Domain": "iot-demo-domain", + "UserPoolId": { + "Ref": "DemoUserPool1AB98549" + } + }, + "Metadata": { + "aws:cdk:path": "OpenSearchPatternStack/DemoUserPool/iot-demo-domain/Resource" + } + }, + "UserPoolUserCreationCustomResourcePolicy1AE20E41": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "cognito-idp:*", + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "UserPoolUserCreationCustomResourcePolicy1AE20E41", + "Roles": [ + { + "Ref": "AWS679f53fac002430cb0da5b7982bd2287ServiceRoleC1EA0FF2" + } + ] + }, + "Metadata": { + "aws:cdk:path": "OpenSearchPatternStack/UserPoolUserCreation/CustomResourcePolicy/Resource" + } + }, + "UserPoolUserCreation7E76E291": { + "Type": "Custom::AWS", + "Properties": { + "ServiceToken": { + "Fn::GetAtt": [ + "AWS679f53fac002430cb0da5b7982bd22872D164C4C", + "Arn" + ] + }, + "Create": { + "Fn::Join": [ + "", + [ + "{\"action\":\"adminCreateUser\",\"service\":\"CognitoIdentityServiceProvider\",\"parameters\":{\"UserPoolId\":\"", + { + "Ref": "DemoUserPool1AB98549" + }, + "\",\"Username\":\"admin\",\"TemporaryPassword\":\"Admin123!\"},\"physicalResourceId\":{\"id\":\"userpoolcreateid2022-08-25 16:39:08\"}}" + ] + ] + }, + "InstallLatestAwsSdk": true + }, + "DependsOn": [ + "UserPoolUserCreationCustomResourcePolicy1AE20E41" + ], + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete", + "Metadata": { + "aws:cdk:path": "OpenSearchPatternStack/UserPoolUserCreation/Resource/Default" + } + }, + "AWS679f53fac002430cb0da5b7982bd2287ServiceRoleC1EA0FF2": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + ] + ] + } + ] + }, + "Metadata": { + "aws:cdk:path": "OpenSearchPatternStack/AWS679f53fac002430cb0da5b7982bd2287/ServiceRole/Resource" + } + }, + "AWS679f53fac002430cb0da5b7982bd22872D164C4C": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "S3Key": "6dbd112fe448437b3438da4382c72fccbb7d2ee1543db222620d7447fffebc50.zip" + }, + "Role": { + "Fn::GetAtt": [ + "AWS679f53fac002430cb0da5b7982bd2287ServiceRoleC1EA0FF2", + "Arn" + ] + }, + "Handler": "index.handler", + "Runtime": "nodejs14.x", + "Timeout": 120 + }, + "DependsOn": [ + "AWS679f53fac002430cb0da5b7982bd2287ServiceRoleC1EA0FF2" + ], + "Metadata": { + "aws:cdk:path": "OpenSearchPatternStack/AWS679f53fac002430cb0da5b7982bd2287/Resource", + "aws:asset:path": "asset.6dbd112fe448437b3438da4382c72fccbb7d2ee1543db222620d7447fffebc50", + "aws:asset:is-bundled": false, + "aws:asset:property": "Code" + } + }, + "masterusergroup": { + "Type": "AWS::Cognito::UserPoolGroup", + "Properties": { + "UserPoolId": { + "Ref": "DemoUserPool1AB98549" + }, + "GroupName": "master-user-group", + "RoleArn": { + "Fn::GetAtt": [ + "demoiotopensearchrole", + "Arn" + ] + } + }, + "DependsOn": [ + "demoiotopensearchrole", + "DemoUserPooliotdemodomainEF32B2ED", + "DemoUserPool1AB98549" + ], + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete", + "Metadata": { + "aws:cdk:path": "OpenSearchPatternStack/master-user-group" + } + }, + "CDKmasterusergroupattachement": { + "Type": "AWS::Cognito::UserPoolUserToGroupAttachment", + "Properties": { + "GroupName": "master-user-group", + "Username": "admin", + "UserPoolId": { + "Ref": "DemoUserPool1AB98549" + } + }, + "DependsOn": [ + "DemoUserPooliotdemodomainEF32B2ED", + "DemoUserPool1AB98549", + "masterusergroup", + "UserPoolUserCreationCustomResourcePolicy1AE20E41", + "UserPoolUserCreation7E76E291" + ], + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete", + "Metadata": { + "aws:cdk:path": "OpenSearchPatternStack/CDK_master_user_group_attachement" + } + }, + "CDKCognitoAccessForOpenSearch56BE0308": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "es.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/AmazonOpenSearchServiceCognitoAccess" + ] + ] + } + ] + }, + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete", + "Metadata": { + "aws:cdk:path": "OpenSearchPatternStack/CDKCognitoAccessForOpenSearch/Resource" + } + }, + "opensearchdemodomain3F2B2C09": { + "Type": "AWS::OpenSearchService::Domain", + "Properties": { + "AdvancedSecurityOptions": { + "Enabled": true, + "InternalUserDatabaseEnabled": false, + "MasterUserOptions": { + "MasterUserARN": { + "Fn::GetAtt": [ + "demoiotopensearchrole", + "Arn" + ] + } + } + }, + "ClusterConfig": { + "DedicatedMasterEnabled": false, + "InstanceCount": 3, + "InstanceType": "t3.small.search", + "ZoneAwarenessEnabled": false + }, + "CognitoOptions": { + "Enabled": true, + "IdentityPoolId": { + "Ref": "DemoIdentityPool" + }, + "RoleArn": { + "Fn::GetAtt": [ + "CDKCognitoAccessForOpenSearch56BE0308", + "Arn" + ] + }, + "UserPoolId": { + "Ref": "DemoUserPool1AB98549" + } + }, + "DomainEndpointOptions": { + "EnforceHTTPS": true, + "TLSSecurityPolicy": "Policy-Min-TLS-1-0-2019-07" + }, + "DomainName": "opensearch-demo-domain", + "EBSOptions": { + "EBSEnabled": true, + "VolumeSize": 10, + "VolumeType": "gp2" + }, + "EncryptionAtRestOptions": { + "Enabled": true + }, + "EngineVersion": "OpenSearch_1.2", + "LogPublishingOptions": {}, + "NodeToNodeEncryptionOptions": { + "Enabled": true + } + }, + "DependsOn": [ + "CDKCognitoAccessForOpenSearch56BE0308", + "demoiotopensearchrole", + "DemoIdentityPool", + "DemoUserPooliotdemodomainEF32B2ED", + "DemoUserPool1AB98549" + ], + "UpdatePolicy": { + "EnableVersionUpgrade": true + }, + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete", + "Metadata": { + "aws:cdk:path": "OpenSearchPatternStack/opensearch-demo-domain/Resource" + } + }, + "opensearchdemodomainAccessPolicyCustomResourcePolicy53C39E12": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "es:UpdateDomainConfig", + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "opensearchdemodomain3F2B2C09", + "Arn" + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "opensearchdemodomainAccessPolicyCustomResourcePolicy53C39E12", + "Roles": [ + { + "Ref": "AWS679f53fac002430cb0da5b7982bd2287ServiceRoleC1EA0FF2" + } + ] + }, + "DependsOn": [ + "CDKCognitoAccessForOpenSearch56BE0308", + "demoiotopensearchrole", + "DemoIdentityPool", + "DemoUserPooliotdemodomainEF32B2ED", + "DemoUserPool1AB98549" + ], + "Metadata": { + "aws:cdk:path": "OpenSearchPatternStack/opensearch-demo-domain/AccessPolicy/CustomResourcePolicy/Resource" + } + }, + "opensearchdemodomainAccessPolicy54A708AD": { + "Type": "Custom::OpenSearchAccessPolicy", + "Properties": { + "ServiceToken": { + "Fn::GetAtt": [ + "AWS679f53fac002430cb0da5b7982bd22872D164C4C", + "Arn" + ] + }, + "Create": { + "Fn::Join": [ + "", + [ + "{\"action\":\"updateDomainConfig\",\"service\":\"OpenSearch\",\"parameters\":{\"DomainName\":\"", + { + "Ref": "opensearchdemodomain3F2B2C09" + }, + "\",\"AccessPolicies\":\"{\\\"Statement\\\":[{\\\"Action\\\":\\\"es:ESHttp*\\\",\\\"Effect\\\":\\\"Allow\\\",\\\"Principal\\\":{\\\"AWS\\\":\\\"*\\\"},\\\"Resource\\\":\\\"", + { + "Fn::GetAtt": [ + "opensearchdemodomain3F2B2C09", + "Arn" + ] + }, + "/*\\\"}],\\\"Version\\\":\\\"2012-10-17\\\"}\"},\"outputPaths\":[\"DomainConfig.AccessPolicies\"],\"physicalResourceId\":{\"id\":\"", + { + "Ref": "opensearchdemodomain3F2B2C09" + }, + "AccessPolicy\"}}" + ] + ] + }, + "Update": { + "Fn::Join": [ + "", + [ + "{\"action\":\"updateDomainConfig\",\"service\":\"OpenSearch\",\"parameters\":{\"DomainName\":\"", + { + "Ref": "opensearchdemodomain3F2B2C09" + }, + "\",\"AccessPolicies\":\"{\\\"Statement\\\":[{\\\"Action\\\":\\\"es:ESHttp*\\\",\\\"Effect\\\":\\\"Allow\\\",\\\"Principal\\\":{\\\"AWS\\\":\\\"*\\\"},\\\"Resource\\\":\\\"", + { + "Fn::GetAtt": [ + "opensearchdemodomain3F2B2C09", + "Arn" + ] + }, + "/*\\\"}],\\\"Version\\\":\\\"2012-10-17\\\"}\"},\"outputPaths\":[\"DomainConfig.AccessPolicies\"],\"physicalResourceId\":{\"id\":\"", + { + "Ref": "opensearchdemodomain3F2B2C09" + }, + "AccessPolicy\"}}" + ] + ] + }, + "InstallLatestAwsSdk": true + }, + "DependsOn": [ + "CDKCognitoAccessForOpenSearch56BE0308", + "demoiotopensearchrole", + "DemoIdentityPool", + "DemoUserPooliotdemodomainEF32B2ED", + "DemoUserPool1AB98549", + "opensearchdemodomainAccessPolicyCustomResourcePolicy53C39E12" + ], + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete", + "Metadata": { + "aws:cdk:path": "OpenSearchPatternStack/opensearch-demo-domain/AccessPolicy/Resource/Default" + } + }, + "ClientIdResourceCustomResourcePolicy90E27E2D": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "cognito-idp:ListUserPoolClients", + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "DemoUserPool1AB98549", + "Arn" + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "ClientIdResourceCustomResourcePolicy90E27E2D", + "Roles": [ + { + "Ref": "AWS679f53fac002430cb0da5b7982bd2287ServiceRoleC1EA0FF2" + } + ] + }, + "DependsOn": [ + "opensearchdemodomainAccessPolicyCustomResourcePolicy53C39E12", + "opensearchdemodomainAccessPolicy54A708AD", + "opensearchdemodomain3F2B2C09" + ], + "Metadata": { + "aws:cdk:path": "OpenSearchPatternStack/ClientIdResource/CustomResourcePolicy/Resource" + } + }, + "ClientIdResource92C73354": { + "Type": "Custom::AWS", + "Properties": { + "ServiceToken": { + "Fn::GetAtt": [ + "AWS679f53fac002430cb0da5b7982bd22872D164C4C", + "Arn" + ] + }, + "Create": { + "Fn::Join": [ + "", + [ + "{\"action\":\"listUserPoolClients\",\"service\":\"CognitoIdentityServiceProvider\",\"parameters\":{\"UserPoolId\":\"", + { + "Ref": "DemoUserPool1AB98549" + }, + "\"},\"physicalResourceId\":{\"id\":\"ClientId-iot-demo-domain\"}}" + ] + ] + }, + "InstallLatestAwsSdk": true + }, + "DependsOn": [ + "ClientIdResourceCustomResourcePolicy90E27E2D", + "opensearchdemodomainAccessPolicyCustomResourcePolicy53C39E12", + "opensearchdemodomainAccessPolicy54A708AD", + "opensearchdemodomain3F2B2C09" + ], + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete", + "Metadata": { + "aws:cdk:path": "OpenSearchPatternStack/ClientIdResource/Resource/Default" + } + }, + "CDKIdentityPoolAttachment": { + "Type": "AWS::Cognito::IdentityPoolRoleAttachment", + "Properties": { + "IdentityPoolId": { + "Ref": "DemoIdentityPool" + }, + "RoleMappings": { + "role_mappings_key": { + "AmbiguousRoleResolution": "Deny", + "IdentityProvider": { + "Fn::Join": [ + "", + [ + "cognito-idp.", + { + "Ref": "AWS::Region" + }, + ".amazonaws.com/", + { + "Ref": "DemoUserPool1AB98549" + }, + ":", + { + "Fn::GetAtt": [ + "ClientIdResource92C73354", + "UserPoolClients.0.ClientId" + ] + } + ] + ] + }, + "Type": "Token" + } + }, + "Roles": {} + }, + "DependsOn": [ + "opensearchdemodomainAccessPolicyCustomResourcePolicy53C39E12", + "opensearchdemodomainAccessPolicy54A708AD", + "opensearchdemodomain3F2B2C09" + ], + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete", + "Metadata": { + "aws:cdk:path": "OpenSearchPatternStack/CDKIdentityPoolAttachment" + } + }, + "iottoopensearchloggroup76C0C407": { + "Type": "AWS::Logs::LogGroup", + "Properties": { + "LogGroupName": "iot_to_opensearch_log_group", + "RetentionInDays": 731 + }, + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete", + "Metadata": { + "aws:cdk:path": "OpenSearchPatternStack/iot_to_opensearch_log_group/Resource" + } + }, + "iotcloudwatcherrorlogrole640ECAA3": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "iot.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + }, + "DependsOn": [ + "iottoopensearchloggroup76C0C407" + ], + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete", + "Metadata": { + "aws:cdk:path": "OpenSearchPatternStack/iot_cloudwatch_error_log_role/Resource" + } + }, + "iotcloudwatcherrorlogroleDefaultPolicy51ECCBCB": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "logs:CreateLogStream", + "logs:DescribeLogStreams", + "logs:PutLogEvents" + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "iottoopensearchloggroup76C0C407", + "Arn" + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "iotcloudwatcherrorlogroleDefaultPolicy51ECCBCB", + "Roles": [ + { + "Ref": "iotcloudwatcherrorlogrole640ECAA3" + } + ] + }, + "DependsOn": [ + "iottoopensearchloggroup76C0C407" + ], + "Metadata": { + "aws:cdk:path": "OpenSearchPatternStack/iot_cloudwatch_error_log_role/DefaultPolicy/Resource" + } + }, + "demotoopensearchrule": { + "Type": "AWS::IoT::TopicRule", + "Properties": { + "TopicRulePayload": { + "Actions": [ + { + "OpenSearch": { + "Endpoint": { + "Fn::Join": [ + "", + [ + "https://", + { + "Fn::GetAtt": [ + "opensearchdemodomain3F2B2C09", + "DomainEndpoint" + ] + } + ] + ] + }, + "Id": "${newuuid()}", + "Index": "iot", + "RoleArn": { + "Fn::GetAtt": [ + "demoiotopensearchrole", + "Arn" + ] + }, + "Type": "_doc" + } + } + ], + "AwsIotSqlVersion": "2016-03-23", + "ErrorAction": { + "CloudwatchLogs": { + "LogGroupName": { + "Ref": "iottoopensearchloggroup76C0C407" + }, + "RoleArn": { + "Fn::GetAtt": [ + "iotcloudwatcherrorlogrole640ECAA3", + "Arn" + ] + } + } + }, + "Sql": "SELECT *, parse_time(\"YYYY-MM-dd'T'hh:mm:ss\", timestamp()) as Time FROM 'Opensearch_demo'" + } + }, + "DependsOn": [ + "demoiotopensearchrole", + "iottoopensearchloggroup76C0C407", + "opensearchdemodomainAccessPolicyCustomResourcePolicy53C39E12", + "opensearchdemodomainAccessPolicy54A708AD", + "opensearchdemodomain3F2B2C09" + ], + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete", + "Metadata": { + "aws:cdk:path": "OpenSearchPatternStack/demo_to_opensearch_rule" + } + }, + "CDKMetadata": { + "Type": "AWS::CDK::Metadata", + "Properties": { + "Analytics": "v2:deflate64:H4sIAAAAAAAA/2WRQW/CMAyFfwv3kLH1sDN02jRph6pjZxRSr3i0cVUnQ1XFf5/T0oLYyZ+fZfm95EkniV4tzImXtjguK9zr/tMbe1Qi7XpLpUNPuk+/3XsBzqPvMqJKfTG0A8hg5gleqDbobkf/lbeWQnMrxLqlQV57cXCo5Zy6u5tTBdfpWaGpB29RVxlVaLu4cqGonlVl6n1hJBa6sgJP7jU465GcmkFWJj4rTnaGGTzrdSzS602wR/Abw6CoAcdgWnsQv79oQffXcCPJSSpZ9x9UziknFsvktQhbatDmIRq0gT3VuxaYQmtB7p44HbT8Iqn7VhJPPJiULyslXhyk5Aock2SdP5B7SPSzflwtfhhx2QZ5yxp0PtY/dxFVE/8BAAA=" + }, + "Metadata": { + "aws:cdk:path": "OpenSearchPatternStack/CDKMetadata/Default" + }, + "Condition": "CDKMetadataAvailable" + } + }, + "Conditions": { + "CDKMetadataAvailable": { + "Fn::Or": [ + { + "Fn::Or": [ + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "af-south-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ap-east-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ap-northeast-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ap-northeast-2" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ap-south-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ap-southeast-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ap-southeast-2" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ca-central-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "cn-north-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "cn-northwest-1" + ] + } + ] + }, + { + "Fn::Or": [ + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "eu-central-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "eu-north-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "eu-south-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "eu-west-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "eu-west-2" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "eu-west-3" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "me-south-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "sa-east-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "us-east-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "us-east-2" + ] + } + ] + }, + { + "Fn::Or": [ + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "us-west-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "us-west-2" + ] + } + ] + } + ] + } + }, + "Parameters": { + "BootstrapVersion": { + "Type": "AWS::SSM::Parameter::Value", + "Default": "/cdk-bootstrap/hnb659fds/version", + "Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip]" + } + }, + "Rules": { + "CheckBootstrapVersion": { + "Assertions": [ + { + "Assert": { + "Fn::Not": [ + { + "Fn::Contains": [ + [ + "1", + "2", + "3", + "4", + "5" + ], + { + "Ref": "BootstrapVersion" + } + ] + } + ] + }, + "AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI." + } + ] + } + } +} diff --git a/cloud_templates/user_guides/opensearch_guide.md b/cloud_templates/user_guides/opensearch_guide.md new file mode 100644 index 0000000..3e7c69d --- /dev/null +++ b/cloud_templates/user_guides/opensearch_guide.md @@ -0,0 +1,202 @@ +# Getting started with Amazon OpenSearch service template guide + +## Setting up and prerequisites + +### AWS Account + +If you don't already have an AWS account follow the [Setup Your Environment](https://aws.amazon.com/getting-started/guides/setup-environment/) getting started guide for a quick overview. + +### AWS CloudFormation + +Before you start using AWS CloudFormation, you might need to know what IAM permissions you need, how to start logging AWS CloudFormation API calls, or what endpoints to use. Refer to this [guide](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/settingup.html) to get started with using AWS CloudFormation. + +### AWS CDK + +**Note**: If you are just going to use the sample demo template you can skip this section. + +The AWS Cloud Development Kit (CDK) is an open source software development framework that lets you define your cloud infrastructure as code in one of its supported programming languages. It is intended for moderately to highly experienced AWS users. Refer to this [guide](https://aws.amazon.com/getting-started/guides/setup-cdk/?pg=gs&sec=gtkaws) to get started with AWS CDK. + +___ + +## Template deployment and CloudFormation stack creation + +A template is a JSON or YAML text file that contains the configuration information about the AWS resources you want to create in the [stack](https://docs.aws.amazon.com/cdk/v2/guide/stacks.html). To learn more about how to work with CloudFormation templates refer to [Working with templates](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/template-guide.html) guide. + +You can either use the provided demo template and deploy it directly in the console or customize the template’s resources before deployment using AWS CDK. Based on your decision follow the respective section below. + +### Sample demo template + +By using the sample json template that is provided, you do not need to take any further actions except creating the stack by uploading the template file. For simplicity’s sake, a simple code is provided that you can run on your device. It is an example of multiple devices sending their weather measurements to the cloud through ExpressLink. You can find the code and guide to get it working under `demo/demo_weather_station_code` directory. + +Follow the steps below to create the CloudFormation stack using the sample template file. + +1. Sign in to the AWS Management Console and open [AWS CloudFormation console.](https://console.aws.amazon.com/cloudformation) +2. If this is a new CloudFormation account, choose **Create New Stack**. Otherwise, choose **Create Stack** and then select **with new resources**. +3. In the **Template** section, select **Upload a template file** and upload the json template file. Choose **Next**. +4. In the **Specify Details** section, enter a stack name in the **Name** field. +5. If you want you can add tags to your stack. Otherwise choose **Next**. +6. Review the stack’s settings and then choose **Create.** +7. At this point, you will find the status of your stack to be `CREATE_IN_PROGRESS`. Your stack might take several minutes (~15 min) to get created. See next sections to learn about monitoring your stack creation. + +### Custom template + +If you are interested in using the CloudFormation templates more than just for demo purposes, you probably need to customize the stack’s resources based on your specific use-case. Follow the steps below to do so: + +1. Make sure that you already [set up your AWS CDK](https://aws.amazon.com/getting-started/guides/setup-cdk/?pg=gs&sec=gtkaws) environment. +2. Starting in your current directory, change your directory and go to `aws_cdk/OpenSearchPattern` directory. +3. Just to verify everything is working correctly, list the stacks in your app by running `cdk ls` command. If you don't see `OpensearchPatternStack`, make sure you are currently in `OpenSearchPattern` directory. +4. The structure of the files inside `OpenSearchPattern` is as below: + +[Image: Screen Shot 2022-08-25 at 12.52.09 PM.png] +* `open_search_pattern_stack.py` is the main code of the stack. It is here where the required resources are created. +* `tests/unit/test_open_search_pattern_stack.py` is where the unit tests of the stack is written. The unit tests check + * Right creation of the resources in addition to their properties + * Dependencies between the resources + * Right error handlings in case of input violations +* `cdk.json` tells the CDK Toolkit how to execute your app. Context values are key-value pairs that can be associated with an app, stack, or construct. You can add the context key-values to this file or in command line before synthesizing the template. +* `README.md` is where you can find the detailed instructions on how to get started with the code including: how to synthesize the template, a set of useful commands, stack’s context parameters, and details about the code. +* `cdk.out` is where the synthesized template (in a json format) will be located in. + +1. Run `source .venv/bin/activate` to activate the app's Python virtual environment. +2. Run `python -m pip install -r requirements.txt` and `python -m pip install -r requirements.txt` to install the dependencies. +3. Go through the `README.md` file to learn about the context parameters that need to be set by you prior to deployment. +4. Set the context parameter values either by changing `cdk.json` file or by using the command line. + 1. To create a command line context variable, use the **`--context (-c) option`**, as shown in the following example: `$ cdk cdk synth -c bucket_name=mybucket+Sheet1!A3` + 2. To specify the same context variable and value in the `cdk.json` file, use the following code.` + {"context": { "bucket_name": "mybucket"}` +5. Run `cdk synth` to emit the synthesized CloudFormation template. +6. Run `python -m pytest` to run the unit tests. It is the best practice to run the tests before deploying your template to the cloud. +7. Run `cdk deploy` to deploy the stack to your default AWS account/region. +8. Use the instructions in ***Stack management*** section below to manage your stack creation. + +___ + +## Stack management + +### Viewing CloudFormation stack data and resources + +After deployment, you may need to monitor your created stack and its resources. To do this, your starting point should be AWS CloudFormation. + +1. Sign in to the AWS Management Console and open [AWS CloudFormation console](https://console.aws.amazon.com/cloudformation). +2. Choose **Stacks** tab to view all the available stacks in your account. +3. Find the stack that you just created and select it. +4. To verify that the stack’s creation is done successfully, check if its status is `CREATE_COMPLETE`. To learn more about what each status means refer to [stack status codes](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/cfn-console-view-stack-data-resources.html#cfn-console-view-stack-data-resources-status-codes). +5. You can view the stack information such as its ID, status, policy, rollback configuration, etc under the **Stack info** tab. +6. If you click on the **Events** tab, each major step in the creation of the stack sorted by the time of each event, with latest events on top is displayed. +7. You can also find the resources that are part of the stack under the **Resources** tab. + +There is more information about viewing stack information [here](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/cfn-console-view-stack-data-resources.html#cfn-console-view-stack-data-resources-view-info). + +### Monitoring the generated resources + +If you deploy and create the stack successfully, the following resources must get created under your stack. You can verify their creation by checking the **Resources** tab in your stack. + +|Resourse |Type | +|--- |--- | +|CDKMetadata |[AWS::CDK::Metadata](https://docs.aws.amazon.com/cdk/api/v1/docs/constructs.ConstructMetadata.html) | +|Cognito Identity Pool |[AWS::Cognito::IdentityPool](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-cognito-identitypool.html) | +|Identity Pool Role Attachment |[AWS::Cognito::IdentityPoolRoleAttachment](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-cognito-identitypoolroleattachment.html) | +|Cognito User Pool |[AWS::Cognito::UserPool](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-cognito-userpool.html) | +|Cognito User Pool Domain |[AWS::Cognito::UserPoolDomain](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-cognito-userpooldomain.html) | +|IAM role and policy (master user for fine grained access) |[AWS::IAM::Role](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-role.html) [AWS::IAM::Policy](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-policy.html) | +|User Pool User creation |[Custom::AWS](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/template-custom-resources.html) | +|Cognito User Pool Group |[AWS::Cognito::UserPoolGroup](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-cognito-userpoolgroup.html) | +|User Pool User/Group Attachment |[AWS::Cognito::UserPoolUserToGroupAttachment](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-cognito-userpoolusertogroupattachment.html) | +|IAM role and policy that grant OpenSearch access to Cognito |[AWS::IAM::Role](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-role.html) [AWS::IAM::Policy](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-policy.html) | +|OpenSearch domain |[AWS::OpenSearchService::Domain](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-opensearchservice-domain.html) | +|IoT Rule |[AWS::IoT::TopicRule](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iot-topicrule.html) | +|CloudWatch log group to capture error logs (IoT Core to OpenSearch) |[AWS::Logs::LogGroup](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-logs-loggroup.html) | +|IAM role and policy that grant IoT access to the CloudWatch log groups |[AWS::IAM::Role](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-role.html) [AWS::IAM::Policy](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-policy.html) | + +### Handling stack failures + +If CloudFormation fails to create, update, or delete your stack, you will be able to go through the logs or error messages to learn more about the issue. There are some general methods for troubleshooting a CloudFormation failure. For example, you can follow the steps below to find the issue in the console: + +* Check the status of your stack in [CloudFormation console](https://console.aws.amazon.com/cloudformation/). +* From the **Events** tab, you can see a set of events while the last operation was being done on your stack. +* Find the failure event from the set of events and then check the status reason of that event. The status reason usually gives a good understanding of the issue that caused the failure. + + +In case of failures in stack creations or updates, CloudFormation automatically performs a rollback. However, you can also [add rollback triggers during stack creation or updating](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-rollback-triggers.html#using-cfn-rollback-triggers-create) to further monitor the state of your application. By setting up the rollback triggers if the application breaches the threshold of the alarms you've specified, it will roll back to that operation. + +Finally, this [troubleshooting guide](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/troubleshooting.html#basic-ts-guide) is a helpful resource to refer if there is an issue in your stack. + +### Estimating the cost of the stack + +There is no additional charge for AWS CloudFormation. You pay for AWS resources created using CloudFormation as if you created them by hand. Refer to this [guide](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-paying.html) to learn more about the stack cost estimation functionality. + +___ + +## Ingesting and visualizing your IoT data with the constructed resources + +### Sending data to the cloud from your device + +Now that your stack and all the required resources are created and available, you can start by connecting your device to the cloud and sending your data to the cloud. + +* If you are new to AWS IoT Core, this [guide](https://docs.aws.amazon.com/iot/latest/developerguide/connect-to-iot.html) is a great starting point to connect your device to the cloud. +* After connecting your device to IoT Core, you can use the [MQTT test client](https://docs.aws.amazon.com/iot/latest/developerguide/view-mqtt-messages.html) to monitor the MQTT messages being passed in your AWS account. +* Move to the **Rules** tab under **Message Routing** section in [AWS IoT console](https://console.aws.amazon.com/iot/home). There you can verify the creation of the newly created topic rule and its [opensearch rule action](https://docs.aws.amazon.com/iot/latest/developerguide/opensearch-rule-action.html) which writes data from MQTT messages to the Amazon OpenSearch Service domain. + +### Access data with OpenSearch Dashboards + +* Open [Amazon Opensearch service console](https://console.aws.amazon.com/opensearch). +* From the left navigation pane, choose **Domains.** +* Find the domain that was created by your stack and select it. +* Open the **OpenSearch Dashboards URL** to access the OpenSearch dashboard. Note that for accessing the OpenSearch dashboards, Cognito authentication is used. (You can find more information about it in [Amazon Cognito authentication for OpenSearch Dashboards](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/cognito-auth.html)). To access the username and password of the master user created by the template for you: + * If you used the sample template file, use “admin” as username and “Admin123!” as your temporary password. In the first attempt to log in to the dashboard you will be asked to change your temporary password. + * If you used the AWS CDK to synthesize the customized template, after running the `cdk deploy` command, you should get a temporary password string that you will use for the first log in to the dashboard. Your username is “admin” by default. However, you can change the username of the master user by modifying `cdk.json` file or using the command line. More details on this can be found in the `README.md` in the stack package. + * You should now be successfully logged in to your domain’s OpenSearch dashboard. If there are issues and you cannot log in, check [Troubleshooting](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/handling-errors.html). +* Before you can use OpenSearch Dashboards, you need an index pattern. Dashboards uses index patterns to narrow your analysis to one or more indices. To match the `iot` index that the template creates by default, go to **Stack Management > Index Patterns**, and define an index pattern of `iot*`, and then choose **Next step**. +* Now you can start creating visualizations. Choose **Visualize**, and then add a new visualization. + +### Optional steps + +* Auto-Tune: Auto-Tune in Amazon OpenSearch Service uses performance and usage metrics from your OpenSearch cluster to suggest memory-related configuration changes, including queue and cache sizes and JVM settings on your nodes which improves cluster speed and stability. See [Auto-Tune for Amazon OpenSearch Service](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/auto-tune.html) fore more details. + +### Integrating with dashboards to visualize data + +#### OpenSearch Dashboards + +OpenSearch Dashboards is the default visualization tool for data in OpenSearch. Amazon OpenSearch Service provides an installation of OpenSearch Dashboards with every OpenSearch Service domain. Refer to ***Access data with OpenSearch Dashboards*** section above to find more details of how to access the OpenSearch dashboard of your domain. [OpenSearch Dashboards](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/dashboards.html) is a useful document to refer for more details. + +#### Amazon QuickSight + +Amazon OpenSearch provides direct integration with [Amazon QuickSight](https://aws.amazon.com/quicksight/). Amazon QuickSight is a fast business analytics service you can use to build visualizations, perform ad-hoc analysis, and quickly get business insights from your data. Amazon QuickSight is available in [these regions](https://docs.aws.amazon.com/general/latest/gr/quicksight.html). + +To connect Amazon OpenSearch Service to QuickSight you need to follow these steps: + +1. Navigate to the AWS QuickSight console. +2. If you have never used AWS QuickSight before, you will be asked to sign up. In this case, choose **Standard** tier and your region as your setup. +3. During the sign up phase, give QuickSight access to your Amazon OpenSearch. +4. If you already have an account, give Amazon QuickSight access your OpenSearch by choosing **Admin >** **Manage QuickSight > Security & permissions.** Under QuickSight access to AWS services, choose **Add or remove**, then select the check box next to **OpenSearch** and choose **Update**. +5. From the admin Amazon QuickSight console page choose **New Analysis** and **New data set.** +6. Choose Amazon OpenSearch Service as the source and enter a name for your data source. Choose the connection type you want to use. (If you are using the sample demo template then leave it as **public**) and then choose your domain. +7. Choose **Validate connection** to check that you can successfully connect to OpenSearch Service. +8. Choose **Create data source** to proceed. +9. After your data source is created, you can start making visualizations in Amazon QuickSight. + +Refer to [Using Amazon OpenSearch Service with Amazon QuickSight](https://docs.aws.amazon.com/quicksight/latest/user/connecting-to-os.html) for more details on QuickSight integration with OpenSearch. + +___ + +## Cleaning up the stack + +To clean-up all the resources used in this demo, all you need to do is to delete the initial CloudFormation stack. To delete a stack and its resources, follow these steps: + +1. Open [AWS CloudFormation console](https://console.aws.amazon.com/cloudformation/). +2. On the Stacks page in the CloudFormation console, select the stack that you want to delete. Note that the stack must be currently running. +3. In the stack details pane, choose **Delete**. +4. Confirm deleting stack when prompted. + +After the stack is deleted, the stack’s status will be `DELETE_COMPLETE`. Stacks in the `DELETE_COMPLETE` state aren't displayed in the CloudFormation console by default. However, you can follow the instructions in [viewing deleted stacks on the AWS CloudFormation console](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/cfn-console-view-deleted-stacks.html) to be able to view them. + +Finally, if the stack deletion failed, the stack will be in the `DELETE_FAILED` state. For solutions, see the [Delete stack fails](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/troubleshooting.html#troubleshooting-errors-delete-stack-fails) troubleshooting topic. In this case, make sure to refer to the **Monitoring the generated resources** section of this document to verify that all the resources got deleted successfully. + +___ + +## Useful resources + +* [CloudFormation User Guide](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/index.html) +* [Amazon OpenSearch User Guide](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/index.html) +* [IoT Core User Guide](https://docs.aws.amazon.com/iot/latest/developerguide/index.html) +* [AWS CDK (v2) User Guide](https://docs.aws.amazon.com/cdk/v2/guide/index.html)