From ef59d08508e21225dcdf4ec2bebcdb362ef419ca Mon Sep 17 00:00:00 2001 From: Mohanna Shahrad <59365092+mohannashahrad@users.noreply.github.com> Date: Wed, 24 Aug 2022 14:47:31 -0400 Subject: [PATCH] Deletes the massive directory --- .../AWS_CDK/IoTAnalyticsPattern/.gitignore | 9 - .../AWS_CDK/IoTAnalyticsPattern/README.md | 105 --- .../AWS_CDK/IoTAnalyticsPattern/app.py | 27 - .../AWS_CDK/IoTAnalyticsPattern/cdk.json | 45 - .../io_t_analytics_pattern/__init__.py | 0 .../io_t_analytics_pattern_stack.py | 201 ----- .../IoTAnalyticsPattern/requirements-dev.txt | 1 - .../IoTAnalyticsPattern/requirements.txt | 2 - .../AWS_CDK/IoTAnalyticsPattern/source.bat | 13 - .../IoTAnalyticsPattern/tests/__init__.py | 0 .../tests/unit/__init__.py | 0 .../unit/test_io_t_analytics_pattern_stack.py | 345 -------- .../AWS_CDK/KinesisPattern/.gitignore | 9 - .../AWS_CDK/KinesisPattern/README.md | 111 --- Cloud_Templates/AWS_CDK/KinesisPattern/app.py | 27 - .../AWS_CDK/KinesisPattern/cdk.json | 47 -- .../kinesis_pattern/__init__.py | 0 .../kinesis_pattern/kinesis_pattern_stack.py | 233 ----- .../KinesisPattern/requirements-dev.txt | 1 - .../AWS_CDK/KinesisPattern/requirements.txt | 2 - .../AWS_CDK/KinesisPattern/source.bat | 13 - .../AWS_CDK/KinesisPattern/tests/__init__.py | 0 .../KinesisPattern/tests/unit/__init__.py | 0 .../tests/unit/test_kinesis_pattern_stack.py | 552 ------------ .../AWS_CDK/OpenSearchPattern/.gitignore | 9 - .../AWS_CDK/OpenSearchPattern/README.md | 139 --- .../AWS_CDK/OpenSearchPattern/app.py | 27 - .../AWS_CDK/OpenSearchPattern/cdk.json | 48 -- .../open_search_pattern/__init__.py | 0 .../open_search_pattern_stack.py | 370 -------- .../OpenSearchPattern/requirements-dev.txt | 1 - .../OpenSearchPattern/requirements.txt | 2 - .../AWS_CDK/OpenSearchPattern/source.bat | 13 - .../OpenSearchPattern/tests/__init__.py | 0 .../OpenSearchPattern/tests/unit/__init__.py | 0 .../unit/test_open_search_pattern_stack.py | 798 ------------------ .../AWS_CDK/TimestreamPattern/.gitignore | 9 - .../AWS_CDK/TimestreamPattern/README.md | 102 --- .../AWS_CDK/TimestreamPattern/app.py | 27 - .../AWS_CDK/TimestreamPattern/cdk.json | 44 - .../TimestreamPattern/requirements-dev.txt | 1 - .../TimestreamPattern/requirements.txt | 2 - .../AWS_CDK/TimestreamPattern/source.bat | 13 - .../TimestreamPattern/tests/__init__.py | 0 .../TimestreamPattern/tests/unit/__init__.py | 0 .../unit/test_timestream_pattern_stack.py | 339 -------- .../timestream_pattern/__init__.py | 0 .../timestream_pattern_stack.py | 160 ---- 48 files changed, 3847 deletions(-) delete mode 100644 Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/.gitignore delete mode 100644 Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/README.md delete mode 100644 Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/app.py delete mode 100644 Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/cdk.json delete mode 100644 Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/io_t_analytics_pattern/__init__.py delete mode 100644 Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/io_t_analytics_pattern/io_t_analytics_pattern_stack.py delete mode 100644 Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/requirements-dev.txt delete mode 100644 Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/requirements.txt delete mode 100644 Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/source.bat delete mode 100644 Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/tests/__init__.py delete mode 100644 Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/tests/unit/__init__.py delete mode 100644 Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/tests/unit/test_io_t_analytics_pattern_stack.py delete mode 100644 Cloud_Templates/AWS_CDK/KinesisPattern/.gitignore delete mode 100644 Cloud_Templates/AWS_CDK/KinesisPattern/README.md delete mode 100644 Cloud_Templates/AWS_CDK/KinesisPattern/app.py delete mode 100644 Cloud_Templates/AWS_CDK/KinesisPattern/cdk.json delete mode 100644 Cloud_Templates/AWS_CDK/KinesisPattern/kinesis_pattern/__init__.py delete mode 100644 Cloud_Templates/AWS_CDK/KinesisPattern/kinesis_pattern/kinesis_pattern_stack.py delete mode 100644 Cloud_Templates/AWS_CDK/KinesisPattern/requirements-dev.txt delete mode 100644 Cloud_Templates/AWS_CDK/KinesisPattern/requirements.txt delete mode 100644 Cloud_Templates/AWS_CDK/KinesisPattern/source.bat delete mode 100644 Cloud_Templates/AWS_CDK/KinesisPattern/tests/__init__.py delete mode 100644 Cloud_Templates/AWS_CDK/KinesisPattern/tests/unit/__init__.py delete mode 100644 Cloud_Templates/AWS_CDK/KinesisPattern/tests/unit/test_kinesis_pattern_stack.py delete mode 100644 Cloud_Templates/AWS_CDK/OpenSearchPattern/.gitignore delete mode 100644 Cloud_Templates/AWS_CDK/OpenSearchPattern/README.md delete mode 100644 Cloud_Templates/AWS_CDK/OpenSearchPattern/app.py delete mode 100644 Cloud_Templates/AWS_CDK/OpenSearchPattern/cdk.json delete mode 100644 Cloud_Templates/AWS_CDK/OpenSearchPattern/open_search_pattern/__init__.py delete mode 100644 Cloud_Templates/AWS_CDK/OpenSearchPattern/open_search_pattern/open_search_pattern_stack.py delete mode 100644 Cloud_Templates/AWS_CDK/OpenSearchPattern/requirements-dev.txt delete mode 100644 Cloud_Templates/AWS_CDK/OpenSearchPattern/requirements.txt delete mode 100644 Cloud_Templates/AWS_CDK/OpenSearchPattern/source.bat delete mode 100644 Cloud_Templates/AWS_CDK/OpenSearchPattern/tests/__init__.py delete mode 100644 Cloud_Templates/AWS_CDK/OpenSearchPattern/tests/unit/__init__.py delete mode 100644 Cloud_Templates/AWS_CDK/OpenSearchPattern/tests/unit/test_open_search_pattern_stack.py delete mode 100644 Cloud_Templates/AWS_CDK/TimestreamPattern/.gitignore delete mode 100644 Cloud_Templates/AWS_CDK/TimestreamPattern/README.md delete mode 100644 Cloud_Templates/AWS_CDK/TimestreamPattern/app.py delete mode 100644 Cloud_Templates/AWS_CDK/TimestreamPattern/cdk.json delete mode 100644 Cloud_Templates/AWS_CDK/TimestreamPattern/requirements-dev.txt delete mode 100644 Cloud_Templates/AWS_CDK/TimestreamPattern/requirements.txt delete mode 100644 Cloud_Templates/AWS_CDK/TimestreamPattern/source.bat delete mode 100644 Cloud_Templates/AWS_CDK/TimestreamPattern/tests/__init__.py delete mode 100644 Cloud_Templates/AWS_CDK/TimestreamPattern/tests/unit/__init__.py delete mode 100644 Cloud_Templates/AWS_CDK/TimestreamPattern/tests/unit/test_timestream_pattern_stack.py delete mode 100644 Cloud_Templates/AWS_CDK/TimestreamPattern/timestream_pattern/__init__.py delete mode 100644 Cloud_Templates/AWS_CDK/TimestreamPattern/timestream_pattern/timestream_pattern_stack.py diff --git a/Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/.gitignore b/Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/.gitignore deleted file mode 100644 index 3037faa..0000000 --- a/Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/.gitignore +++ /dev/null @@ -1,9 +0,0 @@ -*.swp -__pycache__ -.pytest_cache -.venv -*.egg-info - -# CDK asset staging directory -.cdk.staging -cdk.out diff --git a/Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/README.md b/Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/README.md deleted file mode 100644 index 17c7f9b..0000000 --- a/Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/README.md +++ /dev/null @@ -1,105 +0,0 @@ - -# Welcome to your CDK project! -# IoT Data visulaization with AWS IoT Analytics - -The `cdk.json` file tells the CDK Toolkit how to execute your app. - -This project is set up like a standard Python project. The initialization -process also creates a virtualenv within this project, stored under the `.venv` -directory. To create the virtualenv it assumes that there is a `python3` -(or `python` for Windows) executable in your path with access to the `venv` -package. If for any reason the automatic creation of the virtualenv fails, -you can create the virtualenv manually. - -To manually create a virtualenv on MacOS and Linux: - -``` -$ python3 -m venv .venv -``` - -After the init process completes and the virtualenv is created, you can use the following -step to activate your virtualenv. - -``` -$ source .venv/bin/activate -``` - -If you are a Windows platform, you would activate the virtualenv like this: - -``` -% .venv\Scripts\activate.bat -``` - -Once the virtualenv is activated, you can install the required dependencies. - -``` -$ pip install -r requirements.txt -``` - -At this point you can now synthesize the CloudFormation template for this code. - -``` -$ cdk synth -``` - -To add additional dependencies, for example other CDK libraries, just add -them to your `setup.py` file and rerun the `pip install -r requirements.txt` -command. - -## Useful commands - - * `cdk ls` list all stacks in the app - * `cdk synth` emits the synthesized CloudFormation template - * `cdk deploy` deploy this stack to your default AWS account/region - * `cdk diff` compare deployed stack with current state - * `cdk docs` open CDK documentation - -## Context parameters -There are multiple context parameters that you need to set before synthesizing or delpoying this CDK stack. You can specify a context variable either as part of an AWS CDK CLI command, or in `cdk.json`. -To create a command line context variable, use the __--context (-c) option__, as shown in the following example. - -``` -$ cdk cdk synth -c bucket_name=mybucket -``` - -To specify the same context variable and value in the cdk.json file, use the following code. - -``` -{ - "context": { - "bucket_name": "mybucket" - } -} -``` - -In this project, these are the following parameters to be set: - -* `topic_sql` -
It is required for IoT Core rule creation to add a simplified SQL syntax to filter messages received on an MQTT topic and push the data elsewhere. -
__Format__: Enter an SQL statement using the following: ```SELECT FROM WHERE ```. For example: ```SELECT temperature FROM 'iot/topic' WHERE temperature > 50```. To learn more, see AWS IoT SQL Reference. - -* `analytics_channel_name`    `` -
The name of the IoT Analytics channel that will get connected to the IoT Core to get your data. -
__Format__: Choose a unique name that you can easily identify. The channel name must contain 1-128 characters. Valid characters are a-z, A-Z, 0-9, and _ (underscore). - -* `analytics_datastore_name`    `` -
The name of the IoT Analytics datastore that will get connected to the IoT Core to store your data. -
__Format__: A unique ID identifies your data store. You can't change this ID after you create it. Valid characters: a-z, A-Z, 0-9, and _ (underscore). - -* `analytics_dataset_name`    `` -
The name of the IoT Analytics SQL dataset that will get connected to the IoT Core. A SQL dataset is a materialized view from a data store. -
__Format__: Choose a unique name that you can easily identify. The dataset name must contain 1-128 characters. Valid characters are a-z, A-Z, 0-9, and _ (underscore). - -* `analytics_pipeline_name`    `` -
The name of the IoT Analytics pipeline that will read messages from the channel and write processed data to the datastore. -
__Format__: Valid characters: a-z, A-Z, 0-9, and _ (underscore). - -* `analytics_iot_rule_name`    `` -
The name of the IoT Core rule that is going to be created. -
__Format__: Should be an alphanumeric string that can also contain underscore (_) characters, but no spaces. - -* `analytics_iot_role_name`    `` -
An IAM role should be created to grant AWS IoT access to your endpoint. This parameter is for setting the name of this role. -
__Format__: Enter a unique role name that contains alphanumeric characters, hyphens, and underscores. A role name can't contain any spaces. - -Enjoy! diff --git a/Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/app.py b/Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/app.py deleted file mode 100644 index 6b54549..0000000 --- a/Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/app.py +++ /dev/null @@ -1,27 +0,0 @@ -import os - -import aws_cdk as cdk - -from io_t_analytics_pattern.io_t_analytics_pattern_stack import IoTAnalyticsPatternStack - - -app = cdk.App() -IoTAnalyticsPatternStack(app, "IoTAnalyticsPatternStack", - # If you don't specify 'env', this stack will be environment-agnostic. - # Account/Region-dependent features and context lookups will not work, - # but a single synthesized template can be deployed anywhere. - - # Uncomment the next line to specialize this stack for the AWS Account - # and Region that are implied by the current CLI configuration. - - #env=cdk.Environment(account=os.getenv('CDK_DEFAULT_ACCOUNT'), region=os.getenv('CDK_DEFAULT_REGION')), - - # Uncomment the next line if you know exactly what Account and Region you - # want to deploy the stack to. */ - - #env=cdk.Environment(account='123456789012', region='us-east-1'), - - # For more information, see https://docs.aws.amazon.com/cdk/latest/guide/environments.html - ) - -app.synth() diff --git a/Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/cdk.json b/Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/cdk.json deleted file mode 100644 index 7206784..0000000 --- a/Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/cdk.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "app": "python3 app.py", - "watch": { - "include": [ - "**" - ], - "exclude": [ - "README.md", - "cdk*.json", - "requirements*.txt", - "source.bat", - "**/__init__.py", - "python/__pycache__", - "tests" - ] - }, - "context": { - "@aws-cdk/aws-apigateway:usagePlanKeyOrderInsensitiveId": true, - "@aws-cdk/core:stackRelativeExports": true, - "@aws-cdk/aws-rds:lowercaseDbIdentifier": true, - "@aws-cdk/aws-lambda:recognizeVersionProps": true, - "@aws-cdk/aws-lambda:recognizeLayerVersion": true, - "@aws-cdk/aws-cloudfront:defaultSecurityPolicyTLSv1.2_2021": true, - "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true, - "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true, - "@aws-cdk/core:checkSecretUsage": true, - "@aws-cdk/aws-iam:minimizePolicies": true, - "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true, - "@aws-cdk/core:validateSnapshotRemovalPolicy": true, - "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true, - "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true, - "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true, - "@aws-cdk/core:target-partitions": [ - "aws", - "aws-cn" - ], - "topic_sql": "SELECT * FROM 'IoT_Analytics_demo'", - "analytics_channel_name": "demo_iot_channel", - "analytics_datastore_name": "demo_iot_datastore", - "analytics_dataset_name": "demo_iot_dataset", - "analytics_pipeline_name": "demo_iot_pipeline", - "analytics_iot_role_name": "demo_iot_iotanalytics_role", - "analytics_iot_rule_name": "demo_to_iotanalytics_rule" - } -} diff --git a/Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/io_t_analytics_pattern/__init__.py b/Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/io_t_analytics_pattern/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/io_t_analytics_pattern/io_t_analytics_pattern_stack.py b/Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/io_t_analytics_pattern/io_t_analytics_pattern_stack.py deleted file mode 100644 index 38eca33..0000000 --- a/Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/io_t_analytics_pattern/io_t_analytics_pattern_stack.py +++ /dev/null @@ -1,201 +0,0 @@ -from aws_cdk import ( - Stack, - aws_iam as iam, - aws_iot as iot, - aws_iotanalytics as iotanalytics, - aws_s3 as s3, - aws_logs as logs -) -from constructs import Construct -import aws_cdk as cdk -import re -import sys - -sys.path.append('../') - -from customExceptions import * - -class IoTAnalyticsPatternStack(Stack): - - # Defining the class variables - topic_sql = "" - analytics_channel_name = "" - analytics_datastore_name = "" - analytics_dataset_name = "" - analytics_pipeline_name = "" - analytics_iot_role_name = "" - analytics_iot_rule_name = "" - - def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: - super().__init__(scope, construct_id, **kwargs) - - # Getting the context parameters - - # Required parameters for users to set in the CLI command or cdk.json - self.topic_sql = self.node.try_get_context("topic_sql") - - # Optional parameters for users to set in the CLI command or cdk.json - self.analytics_channel_name = self.node.try_get_context("analytics_channel_name") - self.analytics_datastore_name = self.node.try_get_context("analytics_datastore_name") - self.analytics_dataset_name = self.node.try_get_context("analytics_dataset_name") - self.analytics_pipeline_name = self.node.try_get_context("analytics_pipeline_name") - self.analytics_iot_role_name = self.node.try_get_context("analytics_iot_role_name") - self.analytics_iot_rule_name = self.node.try_get_context("analytics_iot_rule_name") - - # Perform input validation - self.performInputValidation() - - # Creating an IoT Analytics Channel - analytics_channel = iotanalytics.CfnChannel(self, self.analytics_channel_name, channel_name=self.analytics_channel_name) - analytics_channel.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) - - # Creating an IoT Analytics Datastore - analytics_datastore = iotanalytics.CfnDatastore(self, self.analytics_datastore_name, datastore_name=self.analytics_datastore_name, - datastore_storage=iotanalytics.CfnDatastore.DatastoreStorageProperty( - service_managed_s3={} - ), - retention_period=iotanalytics.CfnDatastore.RetentionPeriodProperty( - number_of_days=30, - unlimited=False - )) - analytics_datastore.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) - - # Creating an IoT Analytics Dataset - analytics_dataset = iotanalytics.CfnDataset(self, self.analytics_dataset_name, actions=[iotanalytics.CfnDataset.ActionProperty( - action_name="QueryDatastoreCDK", - query_action=iotanalytics.CfnDataset.QueryActionProperty( - sql_query= f'''SELECT * FROM {analytics_datastore.datastore_name}''' - ) - )]) - analytics_dataset.node.add_dependency(analytics_datastore) - analytics_dataset.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) - - # Creating an Iot Analytics Pipeline - analytics_pipeline = iotanalytics.CfnPipeline(self, self.analytics_pipeline_name, pipeline_name=self.analytics_pipeline_name, pipeline_activities=[ - iotanalytics.CfnPipeline.ActivityProperty( - channel=iotanalytics.CfnPipeline.ChannelProperty( - channel_name=analytics_channel.channel_name, - name=analytics_channel.channel_name, - next=analytics_datastore.datastore_name - ), - datastore=iotanalytics.CfnPipeline.DatastoreProperty( - datastore_name=analytics_datastore.datastore_name, - name=analytics_datastore.datastore_name - ) - )]) - analytics_pipeline.node.add_dependency(analytics_datastore) - analytics_pipeline.node.add_dependency(analytics_channel) - analytics_pipeline.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) - - - # Creating the role for the IoT-Analytics rule - channel_arn = f"arn:aws:iotanalytics:{self.region}:{self.account}:channel/{analytics_channel.channel_name}" - iot_analytics_role = iam.Role(self, self.analytics_iot_role_name, assumed_by=iam.ServicePrincipal("iot.amazonaws.com")) - iot_analytics_role.add_to_policy(iam.PolicyStatement(effect=iam.Effect.ALLOW, resources=[channel_arn], actions=["iotanalytics:BatchPutMessage"])) - iot_analytics_role.node.add_dependency(analytics_channel) - iot_analytics_role.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) - - # Creating a cloudwatch log group for topic rule's error action - log_group = logs.LogGroup(self, "iot_to_analytics_log_group" , log_group_name="iot_to_analytics_log_group", removal_policy=cdk.RemovalPolicy.DESTROY) - - iot_to_cloudwatch_logs_role = iam.Role(self, "iot_to_analytics_log_group_role", assumed_by=iam.ServicePrincipal("iot.amazonaws.com")) - iot_to_cloudwatch_logs_role.add_to_policy(iam.PolicyStatement( - effect=iam.Effect.ALLOW, resources=[log_group.log_group_arn], - actions=["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents", "logs:PutMetricFilter", "logs:PutRetentionPolicy"])) - iot_to_cloudwatch_logs_role.node.add_dependency(log_group) - iot_to_cloudwatch_logs_role.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) - - - # Creating the IoT Core Rule - topic_rule = iot.CfnTopicRule(self, self.analytics_iot_rule_name, topic_rule_payload=iot.CfnTopicRule.TopicRulePayloadProperty( - actions=[iot.CfnTopicRule.ActionProperty( iot_analytics=iot.CfnTopicRule.IotAnalyticsActionProperty( - channel_name=analytics_channel.channel_name, - role_arn=iot_analytics_role.role_arn, - ) - )], - sql=self.topic_sql, - error_action= iot.CfnTopicRule.ActionProperty( - cloudwatch_logs=iot.CfnTopicRule.CloudwatchLogsActionProperty( - log_group_name=log_group.log_group_name, - role_arn=iot_to_cloudwatch_logs_role.role_arn - ) - ))) - - topic_rule.node.add_dependency(analytics_channel) - topic_rule.node.add_dependency(iot_analytics_role) - topic_rule.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) - - - def performInputValidation(self): - self.validateTopicSQL(self.topic_sql) - self.validateAnalyticsChannelName(self.analytics_channel_name) - self.validateAnalyticsDatasetName(self.analytics_dataset_name) - self.validateAnalyticsDatastoreName(self.analytics_datastore_name) - self.validateAnalyticsPipelineName(self.analytics_pipeline_name) - self.validateRoleName(self.analytics_iot_role_name) - self.validateIoTRuleName(self.analytics_iot_rule_name) - - def validateTopicSQL(self, sqlStatement): - if not sqlStatement: - raise NoSQL - elif type(sqlStatement) != str: - raise WrongFormattedInput("The input sql statement does not have a right format. Please refer to README.md for more information.") - return - - def validateAnalyticsChannelName(self, channelName): - if not channelName: - self.analytics_channel_name = "demo_iot_channel" - else: - if len(channelName) < 1 or len(channelName) > 128: - raise WrongLengthForInput("Not a valid input for channel name: The channel name must contain 1-128 characters.") - elif not re.match(r'^[a-zA-Z0-9_]+$', channelName): - raise WrongFormattedInput("String format error for channel name: Valid characters are a-z, A-Z, 0-9, and _ (underscore)") - return - - def validateAnalyticsDatasetName(self, datasetName): - if not datasetName: - self.analytics_dataset_name = "demo_iot_dataset" - else: - if len(datasetName) < 1 or len(datasetName) > 128: - raise WrongLengthForInput("Not a valid input for dataset name: The dataset name must contain 1-128 characters.") - elif not re.match(r'^[a-zA-Z0-9_]+$', datasetName): - raise WrongFormattedInput("String format error for dataset name: Valid characters are a-z, A-Z, 0-9, and _ (underscore)") - return - - def validateAnalyticsDatastoreName(self, datastoreName): - if not datastoreName: - self.analytics_datastore_name = "demo_iot_datastore" - else: - if not re.match(r'^[a-zA-Z0-9_]+$', datastoreName): - raise WrongFormattedInput("String format error for datastore name: Valid characters are a-z, A-Z, 0-9, and _ (underscore)") - return - - def validateAnalyticsPipelineName(self, pipelineName): - if not pipelineName: - self.analytics_pipeline_name = "demo_iot_pipeline" - else: - if not re.match(r'^[a-zA-Z0-9_]+$', pipelineName): - raise WrongFormattedInput("String format error for pipeline name: Valid characters are a-z, A-Z, 0-9, and _ (underscore)") - return - - def validateRoleName(self, roleName): - if not roleName: - self.analytics_iot_role_name = "demo_iot_iotanalytics_role" - elif type(roleName) != str: - raise WrongFormattedInput("The provided input for the IAM role name is not of type string") - elif len(roleName) > 64: - raise WrongLengthForInput("The length of the IAM role name string should not exceed 64 characters.") - elif not re.match(r'^[a-zA-Z0-9+=,@-_\.]+$', roleName): - raise WrongFormattedInput("String format error: The IAM role name should be an alphanumeric string that can also contain '+=,.@-_' characters.") - else: - return - - def validateIoTRuleName(self, ruleName): - if not ruleName: - self.analytics_iot_rule_name = "demo_to_iotanalytics_rule" - elif type(ruleName) != str: - raise WrongFormattedInput("The provided input for topic rule name is not of type string") - elif not re.match(r'^[a-zA-Z0-9_]+$', ruleName): - raise WrongFormattedInput("String format error: The topic rule name should be an alphanumeric string that can also contain underscore (_) characters, but no spaces.") - else: - return \ No newline at end of file diff --git a/Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/requirements-dev.txt b/Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/requirements-dev.txt deleted file mode 100644 index 9270945..0000000 --- a/Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/requirements-dev.txt +++ /dev/null @@ -1 +0,0 @@ -pytest==6.2.5 diff --git a/Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/requirements.txt b/Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/requirements.txt deleted file mode 100644 index 0822bbe..0000000 --- a/Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -aws-cdk-lib==2.37.1 -constructs>=10.0.0,<11.0.0 diff --git a/Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/source.bat b/Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/source.bat deleted file mode 100644 index 9e1a834..0000000 --- a/Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/source.bat +++ /dev/null @@ -1,13 +0,0 @@ -@echo off - -rem The sole purpose of this script is to make the command -rem -rem source .venv/bin/activate -rem -rem (which activates a Python virtualenv on Linux or Mac OS X) work on Windows. -rem On Windows, this command just runs this batch file (the argument is ignored). -rem -rem Now we don't need to document a Windows command for activating a virtualenv. - -echo Executing .venv\Scripts\activate.bat for you -.venv\Scripts\activate.bat diff --git a/Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/tests/__init__.py b/Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/tests/unit/__init__.py b/Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/tests/unit/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/tests/unit/test_io_t_analytics_pattern_stack.py b/Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/tests/unit/test_io_t_analytics_pattern_stack.py deleted file mode 100644 index 5da47d9..0000000 --- a/Cloud_Templates/AWS_CDK/IoTAnalyticsPattern/tests/unit/test_io_t_analytics_pattern_stack.py +++ /dev/null @@ -1,345 +0,0 @@ -import aws_cdk as core -import aws_cdk.assertions as assertions -from aws_cdk.assertions import Match -import pytest - -from io_t_analytics_pattern.io_t_analytics_pattern_stack import IoTAnalyticsPatternStack - -app = core.App(context= {"topic_sql": "SELECT temperature, pressure, humidity FROM 'EL-analytics-test'", - "analytics_channel_name": "cdk_iot_channel", - "analytics_datastore_name": "cdk_iot_datastore", - "analytics_dataset_name": "cdk_iot_dataset", - "analytics_pipeline_name": "cdk_iot_pipeline", - "analytics_iot_role_name": "cdk_iot_analytics_role", - "analytics_iot_rule_name": "cdk_to_analytics_rule"}) - -stack = IoTAnalyticsPatternStack(app, "io-t-analytics-pattern") -template = assertions.Template.from_stack(stack) - -# Defining Capture objects for obtaining values in tests -iam_policy_ref = assertions.Capture() -iam_role_ref = assertions.Capture() -iot_datastore_ref = assertions.Capture() -iot_channel_ref = assertions.Capture() - -# Testing the resources' creation and properties - -def test_analytics_channel_created(): - template.has_resource("AWS::IoTAnalytics::Channel", {"DeletionPolicy":"Delete", "UpdateReplacePolicy":"Delete"}) - template.resource_count_is("AWS::IoTAnalytics::Channel", 1) - -def test_analytics_channel_properties(): - template.has_resource_properties("AWS::IoTAnalytics::Channel", { - "ChannelName": app.node.try_get_context("analytics_channel_name") - }) - -def test_analytics_datastore_created(): - template.has_resource("AWS::IoTAnalytics::Datastore", {"DeletionPolicy":"Delete", "UpdateReplacePolicy":"Delete"}) - template.resource_count_is("AWS::IoTAnalytics::Datastore", 1) - -def test_analytics_datastore_properties(): - template.has_resource_properties("AWS::IoTAnalytics::Datastore", { - "DatastoreName": app.node.try_get_context("analytics_datastore_name"), - "DatastoreStorage": {"ServiceManagedS3": {}}, - "RetentionPeriod": {"NumberOfDays": 30, "Unlimited": False} - }) - -def test_analytics_dataset_created(): - template.has_resource("AWS::IoTAnalytics::Dataset", {"DeletionPolicy":"Delete", "UpdateReplacePolicy":"Delete"}) - template.resource_count_is("AWS::IoTAnalytics::Dataset", 1) - -def test_analytics_dataset_properties(): - template.has_resource_properties("AWS::IoTAnalytics::Dataset", { - "Actions": [{ - "ActionName": "QueryDatastoreCDK", - "QueryAction": { - "SqlQuery": f'SELECT * FROM {app.node.try_get_context("analytics_datastore_name")}' - } - }] - }) - -def test_analytics_pipeline_created(): - template.has_resource("AWS::IoTAnalytics::Pipeline", {"DeletionPolicy":"Delete", "UpdateReplacePolicy":"Delete"}) - template.resource_count_is("AWS::IoTAnalytics::Pipeline", 1) - -def test_analytics_pipeline_properties(): - template.has_resource_properties("AWS::IoTAnalytics::Pipeline", { - "PipelineActivities": [{ - "Channel": { - "ChannelName": app.node.try_get_context("analytics_channel_name"), - "Name": app.node.try_get_context("analytics_channel_name"), - "Next": app.node.try_get_context("analytics_datastore_name") - }, - "Datastore": { - "DatastoreName": app.node.try_get_context("analytics_datastore_name"), - "Name": app.node.try_get_context("analytics_datastore_name") - } - }], - "PipelineName": app.node.try_get_context("analytics_pipeline_name") - }) - -def test_iam_role_properties(): - template.has_resource_properties("AWS::IAM::Role", { - "AssumeRolePolicyDocument": { - "Statement": [{ - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "iot.amazonaws.com" - } - }], - "Version": Match.any_value() - } - }) - -def test_iam_policy_properties(): - template.has_resource_properties("AWS::IAM::Policy", { - "PolicyDocument": { - "Statement": [ - { - "Action": "iotanalytics:BatchPutMessage", - "Effect": "Allow", - "Resource": { - "Fn::Join": [ - "", - [ - "arn:aws:iotanalytics:", - { - "Ref": "AWS::Region" - }, - ":", - { - "Ref": "AWS::AccountId" - }, - f":channel/{app.node.try_get_context('analytics_channel_name')}" - ] - ] - } - } - ], - "Version": Match.any_value() - }, - "PolicyName": iam_policy_ref, - "Roles": [ - { - "Ref": iam_role_ref - } - ] - }) - -def test_iot_topic_rule_created(): - template.has_resource("AWS::IoT::TopicRule", {"DeletionPolicy":"Delete", "UpdateReplacePolicy":"Delete"}) - template.resource_count_is("AWS::IoT::TopicRule", 1) - -def test_iot_topic_rule_properties(): - template.has_resource_properties("AWS::IoT::TopicRule", { - "TopicRulePayload": { - "Actions": [{ - "IotAnalytics": { - "ChannelName": app.node.try_get_context("analytics_channel_name"), - "RoleArn": { - "Fn::GetAtt": [ - iam_role_ref.as_string(), - "Arn" - ] - } - } - }], - "Sql": app.node.try_get_context("topic_sql") - } - }) - -# Testing dependencies between the resources - -def test_dataset_dependencies(): - template.has_resource("AWS::IoTAnalytics::Dataset", { - "DependsOn": [ - iot_datastore_ref - ] - }) - -def test_pipeline_dependencies(): - template.has_resource("AWS::IoTAnalytics::Pipeline", { - "DependsOn": [ - iot_channel_ref, - iot_datastore_ref.as_string() - ] - }) - -def test_iam_role_dependencies(): - template.has_resource("AWS::IAM::Role", { - "DependsOn": [ - iot_channel_ref.as_string() - ] - }) - -def test_iam_policy_dependencies(): - template.has_resource("AWS::IAM::Policy", { - "DependsOn": [ - iot_channel_ref.as_string() - ] - }) - -def test_topic_rule_dependencies(): - template.has_resource("AWS::IoT::TopicRule", { - "DependsOn": [ - iam_policy_ref.as_string(), - iam_role_ref.as_string(), - iot_channel_ref.as_string() - ] - }) - -# Testing input validations - -def test_no_sql(): - test_app = core.App(context= { - "analytics_channel_name": "cdk_iot_channel", - "analytics_datastore_name": "cdk_iot_datastore", - "analytics_dataset_name": "cdk_iot_dataset", - "analytics_pipeline_name": "cdk_iot_pipeline", - "analytics_iot_role_name": "cdk_iot_analytics_role", - "analytics_iot_rule_name": "cdk_to_analytics_rule" - }) - with pytest.raises(Exception, match=r"No sql statemtnt .*"): - stack = IoTAnalyticsPatternStack(test_app, "io-t-analytics-pattern") - template = assertions.Template.from_stack(stack) - -def test_wrong_sql_format(): - test_app = core.App(context= { - "topic_sql": ["SELECT * FROM 'topic"], - "analytics_channel_name": "cdk_iot_channel", - "analytics_datastore_name": "cdk_iot_datastore", - "analytics_dataset_name": "cdk_iot_dataset", - "analytics_pipeline_name": "cdk_iot_pipeline", - "analytics_iot_role_name": "cdk_iot_analytics_role", - "analytics_iot_rule_name": "cdk_to_analytics_rule" - }) - with pytest.raises(Exception, match=r"The input sql statement does not have a right format.*"): - stack = IoTAnalyticsPatternStack(test_app, "io-t-analytics-pattern") - template = assertions.Template.from_stack(stack) - -def test_wrong_format_channel_name(): - test_app = core.App(context= { - "topic_sql": "SELECT * FROM 'topic'", - "analytics_channel_name": "cdk-iot-channel", - "analytics_datastore_name": "cdk_iot_datastore", - "analytics_dataset_name": "cdk_iot_dataset", - "analytics_pipeline_name": "cdk_iot_pipeline", - "analytics_iot_role_name": "cdk_iot_analytics_role", - "analytics_iot_rule_name": "cdk_to_analytics_rule" - }) - with pytest.raises(Exception, match=r"String format error for channel name: .*"): - stack = IoTAnalyticsPatternStack(test_app, "io-t-analytics-pattern") - template = assertions.Template.from_stack(stack) - -def test_wrong_length_channel_name(): - test_app = core.App(context= { - "topic_sql": "SELECT * FROM 'topic'", - "analytics_channel_name": "x" * 129, - "analytics_datastore_name": "cdk_iot_datastore", - "analytics_dataset_name": "cdk_iot_dataset", - "analytics_pipeline_name": "cdk_iot_pipeline", - "analytics_iot_role_name": "cdk_iot_analytics_role", - "analytics_iot_rule_name": "cdk_to_analytics_rule" - }) - with pytest.raises(Exception, match=r".* The channel name must contain 1-128 characters."): - stack = IoTAnalyticsPatternStack(test_app, "io-t-analytics-pattern") - template = assertions.Template.from_stack(stack) - -def test_wrong_format_dataset_name(): - test_app = core.App(context= { - "topic_sql": "SELECT * FROM 'topic'", - "analytics_channel_name": "cdk_iot_channel", - "analytics_datastore_name": "cdk_iot_datastore", - "analytics_dataset_name": "cdk-iot-dataset", - "analytics_pipeline_name": "cdk_iot_pipeline", - "analytics_iot_role_name": "cdk_iot_analytics_role", - "analytics_iot_rule_name": "cdk_to_analytics_rule" - }) - with pytest.raises(Exception, match=r"String format error for dataset name: .*"): - stack = IoTAnalyticsPatternStack(test_app, "io-t-analytics-pattern") - template = assertions.Template.from_stack(stack) - -def test_wrong_length_dataset_name(): - test_app = core.App(context= { - "topic_sql": "SELECT * FROM 'topic'", - "analytics_channel_name": "cdk_iot_channel", - "analytics_datastore_name": "cdk_iot_datastore", - "analytics_dataset_name": "x" * 129, - "analytics_pipeline_name": "cdk_iot_pipeline", - "analytics_iot_role_name": "cdk_iot_analytics_role", - "analytics_iot_rule_name": "cdk_to_analytics_rule" - }) - with pytest.raises(Exception, match=r".* The dataset name must contain 1-128 characters."): - stack = IoTAnalyticsPatternStack(test_app, "io-t-analytics-pattern") - template = assertions.Template.from_stack(stack) - -def test_wrong_format_datastore_name(): - test_app = core.App(context= { - "topic_sql": "SELECT * FROM 'topic'", - "analytics_channel_name": "cdk_iot_channel", - "analytics_datastore_name": "cdk-iot-datastore", - "analytics_dataset_name": "cdk_iot_dataset", - "analytics_pipeline_name": "cdk_iot_pipeline", - "analytics_iot_role_name": "cdk_iot_analytics_role", - "analytics_iot_rule_name": "cdk_to_analytics_rule" - }) - with pytest.raises(Exception, match=r"String format error for datastore name: .*"): - stack = IoTAnalyticsPatternStack(test_app, "io-t-analytics-pattern") - template = assertions.Template.from_stack(stack) - -def test_wrong_format_pipeline_name(): - test_app = core.App(context= { - "topic_sql": "SELECT * FROM 'topic'", - "analytics_channel_name": "cdk_iot_channel", - "analytics_datastore_name": "cdk_iot_datastore", - "analytics_dataset_name": "cdk_iot_dataset", - "analytics_pipeline_name": "cdk-iot-pipeline", - "analytics_iot_role_name": "cdk_iot_analytics_role", - "analytics_iot_rule_name": "cdk_to_analytics_rule" - }) - with pytest.raises(Exception, match=r"String format error for pipeline name: .*"): - stack = IoTAnalyticsPatternStack(test_app, "io-t-analytics-pattern") - template = assertions.Template.from_stack(stack) - -def test_wrong_format_role_name(): - test_app = core.App(context= { - "topic_sql": "SELECT * FROM 'topic'", - "analytics_channel_name": "cdk_iot_channel", - "analytics_datastore_name": "cdk_iot_datastore", - "analytics_dataset_name": "cdk_iot_dataset", - "analytics_pipeline_name": "cdk_iot_pipeline", - "analytics_iot_role_name": "cdk_iot_!analytics_role", - "analytics_iot_rule_name": "cdk_to_analytics_rule" - }) - with pytest.raises(Exception, match=r".* The IAM role name should be an alphanumeric string .*"): - stack = IoTAnalyticsPatternStack(test_app, "io-t-analytics-pattern") - template = assertions.Template.from_stack(stack) - -def test_wrong_length_role_name(): - test_app = core.App(context= { - "topic_sql": "SELECT * FROM 'topic'", - "analytics_channel_name": "cdk_iot_channel", - "analytics_datastore_name": "cdk_iot_datastore", - "analytics_dataset_name": "cdk_iot_dataset", - "analytics_pipeline_name": "cdk_iot_pipeline", - "analytics_iot_role_name": "x" * 65, - "analytics_iot_rule_name": "cdk_to_analytics_rule" - }) - with pytest.raises(Exception, match=r"The length of the IAM role .*"): - stack = IoTAnalyticsPatternStack(test_app, "io-t-analytics-pattern") - template = assertions.Template.from_stack(stack) - -def test_wrong_format_rule_name(): - test_app = core.App(context= { - "topic_sql": "SELECT * FROM 'topic'", - "analytics_channel_name": "cdk_iot_channel", - "analytics_datastore_name": "cdk_iot_datastore", - "analytics_dataset_name": "cdk_iot_dataset", - "analytics_pipeline_name": "cdk_iot_pipeline", - "analytics_iot_role_name": "cdk_iot_analytics_role", - "analytics_iot_rule_name": "cdk to analytics_rule" - }) - with pytest.raises(Exception, match=r".* The topic rule name should be an alphanumeric string .*"): - stack = IoTAnalyticsPatternStack(test_app, "io-t-analytics-pattern") - template = assertions.Template.from_stack(stack) \ No newline at end of file diff --git a/Cloud_Templates/AWS_CDK/KinesisPattern/.gitignore b/Cloud_Templates/AWS_CDK/KinesisPattern/.gitignore deleted file mode 100644 index 3037faa..0000000 --- a/Cloud_Templates/AWS_CDK/KinesisPattern/.gitignore +++ /dev/null @@ -1,9 +0,0 @@ -*.swp -__pycache__ -.pytest_cache -.venv -*.egg-info - -# CDK asset staging directory -.cdk.staging -cdk.out diff --git a/Cloud_Templates/AWS_CDK/KinesisPattern/README.md b/Cloud_Templates/AWS_CDK/KinesisPattern/README.md deleted file mode 100644 index 0a06eb6..0000000 --- a/Cloud_Templates/AWS_CDK/KinesisPattern/README.md +++ /dev/null @@ -1,111 +0,0 @@ - -# Welcome to your CDK project! -# IoT Data visulaization with Amazon Kinesis - -The `cdk.json` file tells the CDK Toolkit how to execute your app. - -This project is set up like a standard Python project. The initialization -process also creates a virtualenv within this project, stored under the `.venv` -directory. To create the virtualenv it assumes that there is a `python3` -(or `python` for Windows) executable in your path with access to the `venv` -package. If for any reason the automatic creation of the virtualenv fails, -you can create the virtualenv manually. - -To manually create a virtualenv on MacOS and Linux: - -``` -$ python3 -m venv .venv -``` - -After the init process completes and the virtualenv is created, you can use the following -step to activate your virtualenv. - -``` -$ source .venv/bin/activate -``` - -If you are a Windows platform, you would activate the virtualenv like this: - -``` -% .venv\Scripts\activate.bat -``` - -Once the virtualenv is activated, you can install the required dependencies. - -``` -$ pip install -r requirements.txt -``` - -At this point you can now synthesize the CloudFormation template for this code. - -``` -$ cdk synth -``` - -To add additional dependencies, for example other CDK libraries, just add -them to your `setup.py` file and rerun the `pip install -r requirements.txt` -command. - -## Useful commands - - * `cdk ls` list all stacks in the app - * `cdk synth` emits the synthesized CloudFormation template - * `cdk deploy` deploy this stack to your default AWS account/region - * `cdk diff` compare deployed stack with current state - * `cdk docs` open CDK documentation - -## Context parameters -There are multiple context parameters that you need to set before synthesizing or delpoying this CDK stack. You can specify a context variable either as part of an AWS CDK CLI command, or in `cdk.json`. -To create a command line context variable, use the __--context (-c) option__, as shown in the following example. - -``` -$ cdk cdk synth -c bucket_name=mybucket -``` - -To specify the same context variable and value in the cdk.json file, use the following code. - -``` -{ - "context": { - "bucket_name": "mybucket" - } -} -``` - -In this project, these are the following parameters to be set: - -* `topic_sql` -It is required for IoT Core rule creation to add a simplified SQL syntax to filter messages received on an MQTT topic and push the data elsewhere. -
__Format__: Enter an SQL statement using the following: ```SELECT FROM WHERE ```. For example: ```SELECT temperature FROM 'iot/topic' WHERE temperature > 50```. To learn more, see AWS IoT SQL Reference. - -* `kinesis_destination_bucket_name`    `` -To specify the destination settings for your delivery stream, a s3 bucket must be created and this parameter is for setting the name of it. -
__Format__: Bucket name must be unique and must not contain spaces or uppercase letters. [See rules for bucket naming](https://docs.aws.amazon.com/console/s3/bucket-naming) - -* `kinesis_delivery_stream_name`    `` -The name of the Kinesis delivery stream to get your data and send them to the s3 bucket. -
__Format__: Acceptable characters are uppercase and lowercase letters, numbers, underscores, hyphens, and periods. - -* `kinesis_delivery_stream_role_name`    `` -An IAM role should be created to grant Firehose access to your s3 bucket. This parameter is for setting the name of this role. -
__Format__: Enter a unique role name that contains alphanumeric characters, hyphens, and underscores. A role name can't contain any spaces. - -* `kinesis_iot_rule_name`    `` -The name of the IoT Core rule that is going to be created. -
__Format__: Should be an alphanumeric string that can also contain underscore (_) characters, but no spaces. - -* `kinesis_iot_role_name`    `` -An IAM role should be created to grant AWS IoT access to your endpoint. This parameter is for setting the name of this role. -
__Format__: Enter a unique role name that contains alphanumeric characters, hyphens, and underscores. A role name can't contain any spaces. - -* `glue_crawler_name`    `` - A Glue crawler should be created to crawl the data in your s3 bucket. This parameter is for setting the name of this cralwer. - -* `glue_db_name`    `` -A Glue database should be created to connect to the crawler and store data. This parameter is for setting the name of this database. - -* `glue_crawler_role_name`    `` -An IAM role should be created to grant Glue access to your s3 bucket. This parameter is for setting the name of this role. -
__Format__: Enter a unique role name that contains alphanumeric characters, hyphens, and underscores. A role name can't contain any spaces. - -Enjoy! diff --git a/Cloud_Templates/AWS_CDK/KinesisPattern/app.py b/Cloud_Templates/AWS_CDK/KinesisPattern/app.py deleted file mode 100644 index 707bb7e..0000000 --- a/Cloud_Templates/AWS_CDK/KinesisPattern/app.py +++ /dev/null @@ -1,27 +0,0 @@ -import os - -import aws_cdk as cdk - -from kinesis_pattern.kinesis_pattern_stack import KinesisPatternStack - - -app = cdk.App() -KinesisPatternStack(app, "KinesisPatternStack", - # If you don't specify 'env', this stack will be environment-agnostic. - # Account/Region-dependent features and context lookups will not work, - # but a single synthesized template can be deployed anywhere. - - # Uncomment the next line to specialize this stack for the AWS Account - # and Region that are implied by the current CLI configuration. - - #env=cdk.Environment(account=os.getenv('CDK_DEFAULT_ACCOUNT'), region=os.getenv('CDK_DEFAULT_REGION')), - - # Uncomment the next line if you know exactly what Account and Region you - # want to deploy the stack to. */ - - #env=cdk.Environment(account='123456789012', region='us-east-1'), - - # For more information, see https://docs.aws.amazon.com/cdk/latest/guide/environments.html - ) - -app.synth() diff --git a/Cloud_Templates/AWS_CDK/KinesisPattern/cdk.json b/Cloud_Templates/AWS_CDK/KinesisPattern/cdk.json deleted file mode 100644 index be55934..0000000 --- a/Cloud_Templates/AWS_CDK/KinesisPattern/cdk.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "app": "python3 app.py", - "watch": { - "include": [ - "**" - ], - "exclude": [ - "README.md", - "cdk*.json", - "requirements*.txt", - "source.bat", - "**/__init__.py", - "python/__pycache__", - "tests" - ] - }, - "context": { - "@aws-cdk/aws-apigateway:usagePlanKeyOrderInsensitiveId": true, - "@aws-cdk/core:stackRelativeExports": true, - "@aws-cdk/aws-rds:lowercaseDbIdentifier": true, - "@aws-cdk/aws-lambda:recognizeVersionProps": true, - "@aws-cdk/aws-lambda:recognizeLayerVersion": true, - "@aws-cdk/aws-cloudfront:defaultSecurityPolicyTLSv1.2_2021": true, - "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true, - "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true, - "@aws-cdk/core:checkSecretUsage": true, - "@aws-cdk/aws-iam:minimizePolicies": true, - "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true, - "@aws-cdk/core:validateSnapshotRemovalPolicy": true, - "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true, - "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true, - "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true, - "@aws-cdk/core:target-partitions": [ - "aws", - "aws-cn" - ], - "topic_sql": "SELECT * FROM 'Kinesis_demo'", - "kinesis_destination_bucket_name": "demo-kinesis-bucket", - "kinesis_delivery_stream_role_name": "demo_kinesis_delivery_stream_role", - "kinesis_delivery_stream_name": "demo_delivery_stream", - "kinesis_iot_role_name": "demo_iot_kinesis_role", - "kinesis_iot_rule_name": "demo_to_kinesis_rule", - "glue_db_name": "demo_glue_db", - "glue_crawler_role_name": "demo_glue_crawler_role", - "glue_crawler_name": "demo_glue_crawler" - } -} diff --git a/Cloud_Templates/AWS_CDK/KinesisPattern/kinesis_pattern/__init__.py b/Cloud_Templates/AWS_CDK/KinesisPattern/kinesis_pattern/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/Cloud_Templates/AWS_CDK/KinesisPattern/kinesis_pattern/kinesis_pattern_stack.py b/Cloud_Templates/AWS_CDK/KinesisPattern/kinesis_pattern/kinesis_pattern_stack.py deleted file mode 100644 index 18a8b4a..0000000 --- a/Cloud_Templates/AWS_CDK/KinesisPattern/kinesis_pattern/kinesis_pattern_stack.py +++ /dev/null @@ -1,233 +0,0 @@ -from aws_cdk import ( - Stack, - aws_s3 as s3, - aws_kinesisfirehose as kinesisfirehose, - aws_iot as iot, - aws_iam as iam, - aws_logs as logs, - aws_glue as glue -) -from constructs import Construct -import aws_cdk as cdk -import re -import sys - -sys.path.append('../') - -from customExceptions import * - -# Defining class variables -topic_sql = "" -kinesis_destination_bucket_name = "" -kinesis_delivery_stream_role_name = "" -kinesis_delivery_stream_name = "" -kinesis_iot_role_name = "" -kinesis_iot_rule_name = "" -glue_db_name = "" -glue_crawler_role_name = "" -glue_crawler_name = "" - -class KinesisPatternStack(Stack): - - def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: - super().__init__(scope, construct_id, **kwargs) - - # Get the context parameters - - # Required parameters for users to set in the CLI command or cdk.json - self.topic_sql = self.node.try_get_context("topic_sql") - - # Optional parameters for users to set in the CLI command or cdk.json - self.kinesis_destination_bucket_name = self.node.try_get_context("kinesis_destination_bucket_name") - self.kinesis_delivery_stream_role_name = self.node.try_get_context("kinesis_delivery_stream_role_name") - self.kinesis_delivery_stream_name = self.node.try_get_context("kinesis_delivery_stream_name") - self.kinesis_iot_role_name = self.node.try_get_context("kinesis_iot_role_name") - self.kinesis_iot_rule_name = self.node.try_get_context("kinesis_iot_rule_name") - self.glue_db_name = self.node.try_get_context("glue_db_name") - self.glue_crawler_role_name = self.node.try_get_context("glue_crawler_role_name") - self.glue_crawler_name = self.node.try_get_context("glue_crawler_name") - - # Performing input validation before starting resource creation - self.performInputValidation() - - # Create a bucket for as delivery stream's destination - bucket = s3.Bucket(self, self.kinesis_destination_bucket_name, versioned=True, removal_policy=cdk.RemovalPolicy.DESTROY, auto_delete_objects=True) - - # Creating a role for the delivery stream - firehose_role = iam.Role(self, self.kinesis_delivery_stream_role_name, assumed_by=iam.ServicePrincipal("firehose.amazonaws.com")) - firehose_role.add_to_policy(iam.PolicyStatement(effect=iam.Effect.ALLOW, resources=[bucket.bucket_arn, bucket.bucket_arn + "/*"], actions=["s3:AbortMultipartUpload", - "s3:GetBucketLocation","s3:GetObject", "s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:PutObject"])) - firehose_role.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) - - # Creating a cloud watch log group to capture any errors while sending the data from the delivery stream to the bucket - log_group = logs.LogGroup(self, "Kinesis_deliverystream_to_s3_logs" , log_group_name="Kinesis_deliverystream_to_s3_logs", removal_policy=cdk.RemovalPolicy.DESTROY) - log_stream = logs.LogStream(self, "Kinesis_deliverystream_to_s3_log_stream", log_group=log_group ,log_stream_name="Kinesis_deliverystream_to_s3_log_stream", removal_policy=cdk.RemovalPolicy.DESTROY) - - # Creating the delivery stream - delivery_stream = kinesisfirehose.CfnDeliveryStream(self, self.kinesis_delivery_stream_name, delivery_stream_name=self.kinesis_delivery_stream_name, - s3_destination_configuration=kinesisfirehose.CfnDeliveryStream.S3DestinationConfigurationProperty( - bucket_arn=bucket.bucket_arn, - role_arn= firehose_role.role_arn, - # the properties below are optional - cloud_watch_logging_options=kinesisfirehose.CfnDeliveryStream.CloudWatchLoggingOptionsProperty( - enabled=True, - log_group_name=log_group.log_group_name, - log_stream_name=log_stream.log_stream_name - ) - )) - delivery_stream.node.add_dependency(bucket) - delivery_stream.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) - - # Creating the role for the IoT Kinesis rule - iot_kinesis_role = iam.Role(self, self.kinesis_iot_role_name, assumed_by=iam.ServicePrincipal("iot.amazonaws.com")) - iot_kinesis_role.add_to_policy(iam.PolicyStatement(effect=iam.Effect.ALLOW, resources=[delivery_stream.attr_arn], actions=["firehose:PutRecord", "firehose:PutRecordBatch"])) - iot_kinesis_role.node.add_dependency(delivery_stream) - iot_kinesis_role.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) - - # Creating a cloudwatch log group for topic rule's error action - rule_log_group = logs.LogGroup(self, "iot_to_kinesis_log_group" , log_group_name="iot_to_kinesis_log_group", removal_policy=cdk.RemovalPolicy.DESTROY) - - iot_to_cloudwatch_logs_role = iam.Role(self, "iot_to_kinesis_log_group_role", assumed_by=iam.ServicePrincipal("iot.amazonaws.com")) - iot_to_cloudwatch_logs_role.add_to_policy(iam.PolicyStatement( - effect=iam.Effect.ALLOW, resources=[rule_log_group.log_group_arn], - actions=["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents", "logs:PutMetricFilter", "logs:PutRetentionPolicy"])) - iot_to_cloudwatch_logs_role.node.add_dependency(rule_log_group) - iot_to_cloudwatch_logs_role.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) - - # Creating the IoT rule with action = Kinesis Firehose - topic_rule = iot.CfnTopicRule(self, self.kinesis_iot_rule_name, topic_rule_payload=iot.CfnTopicRule.TopicRulePayloadProperty( - actions=[iot.CfnTopicRule.ActionProperty( firehose=iot.CfnTopicRule.FirehoseActionProperty( - delivery_stream_name=self.kinesis_delivery_stream_name, - role_arn=iot_kinesis_role.role_arn, - - # the properties below are optional - batch_mode=False - ) - )], sql=self.topic_sql, - error_action= iot.CfnTopicRule.ActionProperty( - cloudwatch_logs=iot.CfnTopicRule.CloudwatchLogsActionProperty( - log_group_name=rule_log_group.log_group_name, - role_arn=iot_to_cloudwatch_logs_role.role_arn - ) - ))) - topic_rule.node.add_dependency(delivery_stream) - topic_rule.node.add_dependency(iot_kinesis_role) - topic_rule.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) - - #create an Athena/Glue Database - glue_database = glue.CfnDatabase( - self, - id=self.glue_db_name, - catalog_id=cdk.Aws.ACCOUNT_ID, - database_input=glue.CfnDatabase.DatabaseInputProperty( - description=f"Glue database", - name=self.glue_db_name, - ) - ) - glue_database.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) - - # Creating a role for the glue crawler - glue_s3_role = iam.Role(self, self.glue_crawler_role_name, assumed_by=iam.ServicePrincipal("glue.amazonaws.com")) - glue_s3_role.add_to_policy(iam.PolicyStatement(effect=iam.Effect.ALLOW, resources=[bucket.bucket_arn+"*"], actions=["s3:GetObject", "s3:PutObject"])) - glue_s3_role.add_managed_policy(iam.ManagedPolicy.from_aws_managed_policy_name("service-role/AWSGlueServiceRole")) - glue_s3_role.node.add_dependency(bucket) - glue_s3_role.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) - - - # Creating an Athena/Glue crawler for the s3 bucket - s3_path = f"s3://{bucket.bucket_name}" - crawler = glue.CfnCrawler(self, self.glue_crawler_name, role=glue_s3_role.role_arn, targets=glue.CfnCrawler.TargetsProperty( - s3_targets=[glue.CfnCrawler.S3TargetProperty( - path=s3_path - )]), database_name = self.glue_db_name) - crawler.node.add_dependency(glue_s3_role) - crawler.node.add_dependency(glue_database) - crawler.node.add_dependency(bucket) - crawler.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) - - # Performing input validation on the inpute parameters of the stack - def performInputValidation(self): - self.validateTopicSQL(self.topic_sql) - self.validateBucketName(self.kinesis_destination_bucket_name) - self.validateDeliveryStreamName(self.kinesis_delivery_stream_name) - self.validateIoTRuleName(self.kinesis_iot_rule_name) - self.validateGlueDBName(self.glue_db_name) - self.validateGlueCrawlerName(self.glue_crawler_name) - self.validateIAMRoleName(self.glue_crawler_role_name, resource="crawler") - self.validateIAMRoleName(self.kinesis_iot_role_name, resource="iot") - self.validateIAMRoleName(self.kinesis_delivery_stream_role_name, resource="delivery_stream") - - def validateTopicSQL(self, input): - if not input: - raise NoSQL - elif type(input) != str: - raise WrongFormattedInput("The input sql statement does not have a right format. Please refer to README.md for more information.") - return - - - def validateBucketName(self, input): - if not input: - self.kinesis_destination_bucket_name = "demo_kinesis_bucket" - elif type(input) != str: - raise WrongFormattedInput("The provided input for s3 bucket name is not of type string") - elif len(input) < 3 or len(input) > 63: - raise WrongLengthForInput("Bucket names must be between 3 (min) and 63 (max) characters long.") - elif not re.match(r'(?!(^xn--|-s3alias$))^[a-z0-9][a-z0-9-]{1,61}[a-z0-9]$', input): - raise WrongFormattedInput("Wrong formatted input for s3 bucket name. Please refer to the README.md for more information about bucket naming requirements.") - else: - return - - def validateDeliveryStreamName(self, input): - if not input: - self.kinesis_delivery_stream_name = "demo_delivery_stream" - else: - if type(input) != str: - raise WrongFormattedInput("The provided input for delivery stream name is not of type string") - elif not re.match(r'^[a-zA-Z0-9-_\.]+$', input): - raise WrongFormattedInput("String format error for delivery stream name: Acceptable characters are uppercase and lowercase letters, numbers, underscores, hyphens, and periods.") - - - def validateIoTRuleName(self, input): - if not input: - self.kinesis_iot_rule_name = "demo_to_kinesis_rule" - elif type(input) != str: - raise WrongFormattedInput("The provided input for topic rule name is not of type string") - elif not re.match(r'^[a-zA-Z0-9-_\.]+$', input): - raise WrongFormattedInput("String format error: The topic rule name should be an alphanumeric string that can also contain underscore (_) characters, but no spaces.") - else: - return - - - def validateIAMRoleName(self, input, resource): - if not input: - if resource == 'crawler': - self.glue_crawler_role_name = "demo_glue_crawler_role" - elif resource == 'iot': - self.kinesis_iot_role_name = "demo_iot_kinesis_role" - elif resource == 'delivery_stream': - self.kinesis_delivery_stream_role_name = "demo_kinesis_delivery_stream_role" - elif type(input) != str: - raise WrongFormattedInput("The provided input for the IAM role name is not of type string") - elif len(input) > 64: - raise WrongLengthForInput("The length of the IAM role name string should not exceed 64 characters.") - elif not re.match(r'^[a-zA-Z0-9+=,@-_\.]+$', input): - raise WrongFormattedInput("String format error: The IAM role name should be an alphanumeric string that can also contain '+=,.@-_' characters.") - else: - return - - - def validateGlueDBName(self, input): - if not input: - self.glue_db_name = "demo_glue_db" - elif type(input) != str: - raise WrongFormattedInput("The provided input for the Glue database name is not of type string") - elif len(input) > 255: - raise WrongLengthForInput("The length of the Glue database name string should not exceed 255 characters.") - else: - return - - def validateGlueCrawlerName(self, input): - if not input: - self.glue_crawler_name = "demo_glue_crawler" - else: - return \ No newline at end of file diff --git a/Cloud_Templates/AWS_CDK/KinesisPattern/requirements-dev.txt b/Cloud_Templates/AWS_CDK/KinesisPattern/requirements-dev.txt deleted file mode 100644 index 9270945..0000000 --- a/Cloud_Templates/AWS_CDK/KinesisPattern/requirements-dev.txt +++ /dev/null @@ -1 +0,0 @@ -pytest==6.2.5 diff --git a/Cloud_Templates/AWS_CDK/KinesisPattern/requirements.txt b/Cloud_Templates/AWS_CDK/KinesisPattern/requirements.txt deleted file mode 100644 index 0822bbe..0000000 --- a/Cloud_Templates/AWS_CDK/KinesisPattern/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -aws-cdk-lib==2.37.1 -constructs>=10.0.0,<11.0.0 diff --git a/Cloud_Templates/AWS_CDK/KinesisPattern/source.bat b/Cloud_Templates/AWS_CDK/KinesisPattern/source.bat deleted file mode 100644 index 9e1a834..0000000 --- a/Cloud_Templates/AWS_CDK/KinesisPattern/source.bat +++ /dev/null @@ -1,13 +0,0 @@ -@echo off - -rem The sole purpose of this script is to make the command -rem -rem source .venv/bin/activate -rem -rem (which activates a Python virtualenv on Linux or Mac OS X) work on Windows. -rem On Windows, this command just runs this batch file (the argument is ignored). -rem -rem Now we don't need to document a Windows command for activating a virtualenv. - -echo Executing .venv\Scripts\activate.bat for you -.venv\Scripts\activate.bat diff --git a/Cloud_Templates/AWS_CDK/KinesisPattern/tests/__init__.py b/Cloud_Templates/AWS_CDK/KinesisPattern/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/Cloud_Templates/AWS_CDK/KinesisPattern/tests/unit/__init__.py b/Cloud_Templates/AWS_CDK/KinesisPattern/tests/unit/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/Cloud_Templates/AWS_CDK/KinesisPattern/tests/unit/test_kinesis_pattern_stack.py b/Cloud_Templates/AWS_CDK/KinesisPattern/tests/unit/test_kinesis_pattern_stack.py deleted file mode 100644 index bbb2131..0000000 --- a/Cloud_Templates/AWS_CDK/KinesisPattern/tests/unit/test_kinesis_pattern_stack.py +++ /dev/null @@ -1,552 +0,0 @@ -import aws_cdk as core -import aws_cdk.assertions as assertions -from aws_cdk.assertions import Match - -from kinesis_pattern.kinesis_pattern_stack import KinesisPatternStack -import pytest - -# Setting the context for the app -app = core.App(context={"topic_sql": "SELECT temperature, pressure, humidity FROM 'EL-kinesis-test'", - "kinesis_destination_bucket_name": "cdk-kinesis-bucket", - "kinesis_delivery_stream_role_name": "cdk_kinesis_delivery_stream_role", - "kinesis_delivery_stream_name": "cdk_delivery_stream", - "kinesis_iot_role_name": "cdk_iot_kinesis_role", - "kinesis_iot_rule_name": "cdk_to_kinesis_rule", - "glue_db_name": "cdk_glue_db", - "glue_crawler_role_name": "cdk_glue_crawler_role", - "glue_crawler_name": "cdk_glue_crawler"}) - -stack = KinesisPatternStack(app, "kinesis-pattern") -template = assertions.Template.from_stack(stack) - -# Defining Capture objects for obtaining values in tests -bucket_ref_capture = assertions.Capture() -bucket_policy_ref = assertions.Capture() -bucket_auto_delete_object_ref = assertions.Capture() -delivery_stream_policy_name = assertions.Capture() -delivery_stream_role_ref = assertions.Capture() -iot_to_kinesis_role_ref = assertions.Capture() -iot_to_kinesis_role_policy_ref = assertions.Capture() -glue_crawler_role_ref = assertions.Capture() -glue_crawler_role_policy = assertions.Capture() -log_group_ref = assertions.Capture() -delivery_stream_logicalID = assertions.Capture() -glueDB_logicalID = assertions.Capture() - -# Testing the resources' creation and properties - -def test_bucket_creation(): - template.has_resource("AWS::S3::Bucket", {"DeletionPolicy":"Delete", "UpdateReplacePolicy":"Delete"}) - template.resource_count_is("AWS::S3::Bucket",1) - -def test_bucket_properties(): - template.has_resource_properties("AWS::S3::Bucket", { - "VersioningConfiguration": {"Status": "Enabled"} - }) - -def test_bucket_policy_creation(): - template.resource_count_is("AWS::S3::BucketPolicy",1) - -def test_bucket_policy_properties(): - - template.has_resource_properties("AWS::S3::BucketPolicy", { - "Bucket": {"Ref": bucket_ref_capture}, - }) - - template.has_resource_properties("AWS::S3::BucketPolicy", { - "PolicyDocument": { - "Statement": [{ - "Action": [ "s3:GetBucket*", "s3:List*", "s3:DeleteObject*"], - "Effect": "Allow", - "Principal": Match.any_value(), - "Resource": [ - { - "Fn::GetAtt": [ - bucket_ref_capture.as_string(), - "Arn" - ] - }, - { - "Fn::Join": [ - "", - [ { - "Fn::GetAtt": [ - bucket_ref_capture.as_string(), - "Arn" - ] - }, - "/*" - ]] - }] - }], - "Version": Match.any_value() - } - }) - - -def test_delivery_stream_role_properties(): - template.has_resource_properties("AWS::IAM::Role", { - "AssumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "firehose.amazonaws.com" - } - } - ], - "Version": Match.any_value() - } - }) - -def test_delivery_stream_role_policy_properties(): - template.has_resource_properties("AWS::IAM::Policy", { - "PolicyDocument": { - "Statement": [ - { - "Action": [ - "s3:AbortMultipartUpload", - "s3:GetBucketLocation", - "s3:GetObject", - "s3:ListBucket", - "s3:ListBucketMultipartUploads", - "s3:PutObject" - ], - "Effect": "Allow", - "Resource": [ - { "Fn::GetAtt": [ bucket_ref_capture.as_string(), "Arn"]}, - { - "Fn::Join": [ - "", - [{ "Fn::GetAtt": [ bucket_ref_capture.as_string(), "Arn" ]}, - "/*" - ]]} - ] - }], - "Version": Match.any_value() - }, - "PolicyName": delivery_stream_policy_name, - "Roles": [{ "Ref": delivery_stream_role_ref }] - }) - -def test_iot_to_kinesis_role_properties(): - template.has_resource_properties("AWS::IAM::Role", { - "AssumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "iot.amazonaws.com" - } - } - ], - "Version": Match.any_value() - } - }) - -def test_iot_to_kinesis_role_policy_properties(): - template.has_resource_properties("AWS::IAM::Policy", { - "PolicyDocument": { - "Statement": [{ - "Action": [ - "firehose:PutRecord", - "firehose:PutRecordBatch" - ], - "Effect": "Allow", - "Resource": { - "Fn::GetAtt": [ - delivery_stream_logicalID, - "Arn" - ] - } - }], - "Version": Match.any_value() - }, - "Roles": [{ "Ref": iot_to_kinesis_role_ref }] - }) - -def test_log_group_creation(): - template.has_resource("AWS::Logs::LogGroup", {"DeletionPolicy":"Delete", "UpdateReplacePolicy":"Delete"}) - template.resource_count_is("AWS::Logs::LogGroup",2) - -def test_log_group_properties(): - template.has_resource_properties("AWS::Logs::LogGroup", { - "LogGroupName": "Kinesis_deliverystream_to_s3_logs", - "RetentionInDays": 731 - }) - -def test_log_stream_creation(): - template.has_resource("AWS::Logs::LogStream", {"DeletionPolicy":"Delete", "UpdateReplacePolicy":"Delete"}) - template.resource_count_is("AWS::Logs::LogStream",1) - -def test_log_stream_properties(): - template.has_resource_properties("AWS::Logs::LogStream", { - "LogGroupName": { - "Ref": log_group_ref - }, - "LogStreamName": "Kinesis_deliverystream_to_s3_log_stream" - }) - - -def test_delivery_stream_creation(): - template.has_resource("AWS::KinesisFirehose::DeliveryStream", {"DeletionPolicy":"Delete", "UpdateReplacePolicy":"Delete"}) - template.resource_count_is("AWS::KinesisFirehose::DeliveryStream",1) - -def test_delivery_stream_properties(): - template.has_resource_properties("AWS::KinesisFirehose::DeliveryStream", { - "DeliveryStreamName": app.node.try_get_context("kinesis_delivery_stream_name"), - "S3DestinationConfiguration": { - "BucketARN": { - "Fn::GetAtt": [ - bucket_ref_capture.as_string(), - "Arn" - ] - }, - "CloudWatchLoggingOptions": { - "Enabled": True, - "LogGroupName": { - "Ref": log_group_ref.as_string() - }, - "LogStreamName": { - "Ref": Match.any_value() - } - }, - "RoleARN": { - "Fn::GetAtt": [ - delivery_stream_role_ref.as_string(), - "Arn" - ] - } - } - }) - -def test_iot_rule_creation(): - template.has_resource("AWS::IoT::TopicRule", {"DeletionPolicy":"Delete", "UpdateReplacePolicy":"Delete" }) - template.resource_count_is("AWS::IoT::TopicRule",1) - -def test_iot_rule_properties(): - template.has_resource_properties("AWS::IoT::TopicRule", { - "TopicRulePayload": { - "Actions": [ - { - "Firehose": { - "BatchMode": False, - "DeliveryStreamName": app.node.try_get_context("kinesis_delivery_stream_name"), - "RoleArn": { - "Fn::GetAtt": [ - iot_to_kinesis_role_ref.as_string(), - "Arn" - ] - } - } - } - ], - "Sql": app.node.try_get_context("topic_sql") - } - }) - -def test_glue_database_creation(): - template.has_resource("AWS::Glue::Database", {"DeletionPolicy":"Delete", "UpdateReplacePolicy":"Delete"}) - template.resource_count_is("AWS::Glue::Database",1) - -def test_glue_database_properties(): - template.has_resource_properties("AWS::Glue::Database", { - "DatabaseInput": { - "Description": "Glue database", - "Name": app.node.try_get_context("glue_db_name") - } - }) - -def test_glue_crawler_role_properties(): - template.has_resource_properties("AWS::IAM::Role", { - "AssumeRolePolicyDocument": { - "Statement": [{ - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "glue.amazonaws.com" - } - }], - "Version": Match.any_value() - } - }) - -def test_glue_crawler_role_policy_properties(): - template.has_resource_properties("AWS::IAM::Policy", { - "PolicyDocument": { - "Statement": [ - { - "Action": [ - "s3:GetObject", - "s3:PutObject" - ], - "Effect": "Allow", - "Resource": { - "Fn::Join": [ - "", - [ - { - "Fn::GetAtt": [ - bucket_ref_capture.as_string(), - "Arn" - ] - }, - "*" - ] - ] - } - } - ], - "Version": Match.any_value() - }, - "Roles": [{"Ref": glue_crawler_role_ref}] - }) - -def test_glue_crawler_creation(): - template.has_resource("AWS::Glue::Crawler", {"DeletionPolicy":"Delete", "UpdateReplacePolicy":"Delete"}) - template.resource_count_is("AWS::Glue::Crawler",1) - -def test_glue_crawler_properties(): - template.has_resource_properties("AWS::Glue::Crawler", { - "Role": { - "Fn::GetAtt": [ - glue_crawler_role_ref.as_string(), - "Arn" - ] - }, - "Targets": { - "S3Targets": [ - { - "Path": { - "Fn::Join": [ - "", - [ - "s3://", - { - "Ref": bucket_ref_capture.as_string() - } - ] - ] - } - } - ] - }, - "DatabaseName": app.node.try_get_context("glue_db_name") - }) - -# Testing dependencies between the resources - -def test_delivery_stream_dependencies(): - template.has_resource("AWS::KinesisFirehose::DeliveryStream", { - "DependsOn": [ - bucket_auto_delete_object_ref, - bucket_policy_ref, - bucket_ref_capture.as_string() - ] - }) - -def test_delivery_stream_dependencies(): - template.has_resource("AWS::KinesisFirehose::DeliveryStream", { - "DependsOn": [ - bucket_auto_delete_object_ref, - bucket_policy_ref, - bucket_ref_capture.as_string() - ] - }) - -def test_iot_to_kinesis_role_dependencies(): - template.has_resource("AWS::IAM::Role", { - "DependsOn": [ - delivery_stream_logicalID.as_string() - ] - }) - -def test_iot_topic_rule_dependencies(): - template.has_resource("AWS::IoT::TopicRule", { - "DependsOn": [ - delivery_stream_logicalID.as_string(), - iot_to_kinesis_role_policy_ref, - iot_to_kinesis_role_ref.as_string() - ] - }) - -def test_glue_crawler_role_dependencies(): - template.has_resource("AWS::IAM::Role", { - "DependsOn": [ - bucket_auto_delete_object_ref.as_string(), - bucket_policy_ref.as_string(), - bucket_ref_capture.as_string() - ] - }) - -def test_glue_crawler_dependencies(): - template.has_resource("AWS::Glue::Crawler", { - "DependsOn": [ - glue_crawler_role_policy, - glue_crawler_role_ref.as_string(), - glueDB_logicalID, - bucket_auto_delete_object_ref.as_string(), - bucket_policy_ref.as_string(), - bucket_ref_capture.as_string() - ] - }) - -# Testing input validation process - -def test_no_sql(): - test_app = core.App(context= { - "kinesis_destination_bucket_name": "cdk-kinesis-bucket", - "kinesis_delivery_stream_role_name": "cdk_kinesis_delivery_stream_role", - "kinesis_delivery_stream_name": "cdk_delivery_stream", - "kinesis_iot_role_name": "cdk_iot_kinesis_role", - "kinesis_iot_rule_name": "cdk_to_kinesis_rule", - "glue_db_name": "cdk_glue_db", - "glue_crawler_role_name": "cdk_glue_crawler_role", - "glue_crawler_name": "cdk_glue_crawler" - }) - with pytest.raises(Exception, match=r"No sql statemtnt .*"): - stack = KinesisPatternStack(test_app, "kinesis-pattern") - template = assertions.Template.from_stack(stack) - -def test_wrong_sql_format(): - test_app = core.App(context= { - "topic_sql": ["SELECT temperature, pressure, humidity FROM 'EL-kinesis-test'"], - "kinesis_destination_bucket_name": "cdk-kinesis-bucket", - "kinesis_delivery_stream_role_name": "cdk_kinesis_delivery_stream_role", - "kinesis_delivery_stream_name": "cdk_delivery_stream", - "kinesis_iot_role_name": "cdk_iot_kinesis_role", - "kinesis_iot_rule_name": "cdk_to_kinesis_rule", - "glue_db_name": "cdk_glue_db", - "glue_crawler_role_name": "cdk_glue_crawler_role", - "glue_crawler_name": "cdk_glue_crawler" - }) - with pytest.raises(Exception, match=r"The input sql statement does not have a right format. .*"): - stack = KinesisPatternStack(test_app, "kinesis-pattern") - template = assertions.Template.from_stack(stack) - -def test_wrong_bucket_name_format(): - test_app = core.App(context= { - "topic_sql": "SELECT temperature, pressure, humidity FROM 'EL-kinesis-test'", - "kinesis_destination_bucket_name": "cdk_kinesis_bucket", - "kinesis_delivery_stream_role_name": "cdk_kinesis_delivery_stream_role", - "kinesis_delivery_stream_name": "cdk_delivery_stream", - "kinesis_iot_role_name": "cdk_iot_kinesis_role", - "kinesis_iot_rule_name": "cdk_to_kinesis_rule", - "glue_db_name": "cdk_glue_db", - "glue_crawler_role_name": "cdk_glue_crawler_role", - "glue_crawler_name": "cdk_glue_crawler" - }) - with pytest.raises(Exception, match=r"Wrong formatted input for s3 bucket name. .*"): - stack = KinesisPatternStack(test_app, "kinesis-pattern") - template = assertions.Template.from_stack(stack) - - test_app = core.App(context= { - "topic_sql": "SELECT temperature, pressure, humidity FROM 'EL-kinesis-test'", - "kinesis_destination_bucket_name": "xn--bucket", - "kinesis_delivery_stream_role_name": "cdk_kinesis_delivery_stream_role", - "kinesis_delivery_stream_name": "cdk_delivery_stream", - "kinesis_iot_role_name": "cdk_iot_kinesis_role", - "kinesis_iot_rule_name": "cdk_to_kinesis_rule", - "glue_db_name": "cdk_glue_db", - "glue_crawler_role_name": "cdk_glue_crawler_role", - "glue_crawler_name": "cdk_glue_crawler" - }) - with pytest.raises(Exception, match=r"Wrong formatted input for s3 bucket name. .*"): - stack = KinesisPatternStack(test_app, "kinesis-pattern") - template = assertions.Template.from_stack(stack) - - test_app = core.App(context= { - "topic_sql": "SELECT temperature, pressure, humidity FROM 'EL-kinesis-test'", - "kinesis_destination_bucket_name": "x" * 70, - "kinesis_delivery_stream_role_name": "cdk_kinesis_delivery_stream_role", - "kinesis_delivery_stream_name": "cdk_delivery_stream", - "kinesis_iot_role_name": "cdk_iot_kinesis_role", - "kinesis_iot_rule_name": "cdk_to_kinesis_rule", - "glue_db_name": "cdk_glue_db", - "glue_crawler_role_name": "cdk_glue_crawler_role", - "glue_crawler_name": "cdk_glue_crawler" - }) - with pytest.raises(Exception, match=r"Bucket names must be between 3 .*"): - stack = KinesisPatternStack(test_app, "kinesis-pattern") - template = assertions.Template.from_stack(stack) - -def test_wrong_delivery_stream_format(): - test_app = core.App(context= { - "topic_sql": "SELECT temperature, pressure, humidity FROM 'EL-kinesis-test'", - "kinesis_destination_bucket_name": "cdk-kinesis-bucket", - "kinesis_delivery_stream_role_name": "cdk_kinesis_delivery_stream_role", - "kinesis_delivery_stream_name": "cdk_delivery@stream", - "kinesis_iot_role_name": "cdk_iot_kinesis_role", - "kinesis_iot_rule_name": "cdk_to_kinesis_rule", - "glue_db_name": "cdk_glue_db", - "glue_crawler_role_name": "cdk_glue_crawler_role", - "glue_crawler_name": "cdk_glue_crawler" - }) - with pytest.raises(Exception, match=r"String format error for delivery stream name: .*"): - stack = KinesisPatternStack(test_app, "kinesis-pattern") - template = assertions.Template.from_stack(stack) - -def test_wrong_topic_rule_name_format(): - test_app = core.App(context= { - "topic_sql": "SELECT temperature, pressure, humidity FROM 'EL-kinesis-test'", - "kinesis_destination_bucket_name": "cdk-kinesis-bucket", - "kinesis_delivery_stream_role_name": "cdk_kinesis_delivery_stream_role", - "kinesis_delivery_stream_name": "cdk_delivery_stream", - "kinesis_iot_role_name": "cdk_iot_kinesis_role", - "kinesis_iot_rule_name": "cdk_to kinesis_rule", - "glue_db_name": "cdk_glue_db", - "glue_crawler_role_name": "cdk_glue_crawler_role", - "glue_crawler_name": "cdk_glue_crawler" - }) - with pytest.raises(Exception, match=r"String format error: The topic rule name .*"): - stack = KinesisPatternStack(test_app, "kinesis-pattern") - template = assertions.Template.from_stack(stack) - -def test_wrong_iam_role_name_format(): - test_app = core.App(context= { - "topic_sql": "SELECT temperature, pressure, humidity FROM 'EL-kinesis-test'", - "kinesis_destination_bucket_name": "cdk-kinesis-bucket", - "kinesis_delivery_stream_role_name": "cdk_kinesis_delivery_stream_role", - "kinesis_delivery_stream_name": "cdk_delivery_stream", - "kinesis_iot_role_name": "cdk_iot_kinesis_role", - "kinesis_iot_rule_name": "cdk_to_kinesis_rule", - "glue_db_name": "cdk_glue_db", - "glue_crawler_role_name": "cdk_!glue_crawler_role", - "glue_crawler_name": "cdk_glue_crawler" - }) - with pytest.raises(Exception, match=r"String format error: The IAM role name should be .*"): - stack = KinesisPatternStack(test_app, "kinesis-pattern") - template = assertions.Template.from_stack(stack) - - test_app = core.App(context= { - "topic_sql": "SELECT temperature, pressure, humidity FROM 'EL-kinesis-test'", - "kinesis_destination_bucket_name": "cdk-kinesis-bucket", - "kinesis_delivery_stream_role_name": "c" * 70, - "kinesis_delivery_stream_name": "cdk_delivery_stream", - "kinesis_iot_role_name": "cdk_iot_kinesis_role", - "kinesis_iot_rule_name": "cdk_to_kinesis_rule", - "glue_db_name": "cdk_glue_db", - "glue_crawler_role_name": "cdk_glue_crawler_role", - "glue_crawler_name": "cdk_glue_crawler" - }) - with pytest.raises(Exception, match=r"The length of the IAM role name .*"): - stack = KinesisPatternStack(test_app, "kinesis-pattern") - template = assertions.Template.from_stack(stack) - -def test_wrong_glue_db_name_format(): - test_app = core.App(context= { - "topic_sql": "SELECT temperature, pressure, humidity FROM 'EL-kinesis-test'", - "kinesis_destination_bucket_name": "cdk-kinesis-bucket", - "kinesis_delivery_stream_role_name": "cdk_kinesis_delivery_stream_role", - "kinesis_delivery_stream_name": "cdk_delivery_stream", - "kinesis_iot_role_name": "cdk_iot_kinesis_role", - "kinesis_iot_rule_name": "cdk_to_kinesis_rule", - "glue_db_name": "c" * 256, - "glue_crawler_role_name": "cdk_glue_crawler_role", - "glue_crawler_name": "cdk_glue_crawler" - }) - with pytest.raises(Exception, match=r"The length of the Glue database .*"): - stack = KinesisPatternStack(test_app, "kinesis-pattern") - template = assertions.Template.from_stack(stack) \ No newline at end of file diff --git a/Cloud_Templates/AWS_CDK/OpenSearchPattern/.gitignore b/Cloud_Templates/AWS_CDK/OpenSearchPattern/.gitignore deleted file mode 100644 index 3037faa..0000000 --- a/Cloud_Templates/AWS_CDK/OpenSearchPattern/.gitignore +++ /dev/null @@ -1,9 +0,0 @@ -*.swp -__pycache__ -.pytest_cache -.venv -*.egg-info - -# CDK asset staging directory -.cdk.staging -cdk.out diff --git a/Cloud_Templates/AWS_CDK/OpenSearchPattern/README.md b/Cloud_Templates/AWS_CDK/OpenSearchPattern/README.md deleted file mode 100644 index d43eaea..0000000 --- a/Cloud_Templates/AWS_CDK/OpenSearchPattern/README.md +++ /dev/null @@ -1,139 +0,0 @@ - -# Welcome to your CDK project! -# IoT Data visulaization with Amazon Opensearch Service - -The `cdk.json` file tells the CDK Toolkit how to execute your app. - -This project is set up like a standard Python project. The initialization -process also creates a virtualenv within this project, stored under the `.venv` -directory. To create the virtualenv it assumes that there is a `python3` -(or `python` for Windows) executable in your path with access to the `venv` -package. If for any reason the automatic creation of the virtualenv fails, -you can create the virtualenv manually. - -To manually create a virtualenv on MacOS and Linux: - -``` -$ python3 -m venv .venv -``` - -After the init process completes and the virtualenv is created, you can use the following -step to activate your virtualenv. - -``` -$ source .venv/bin/activate -``` - -If you are a Windows platform, you would activate the virtualenv like this: - -``` -% .venv\Scripts\activate.bat -``` - -Once the virtualenv is activated, you can install the required dependencies. - -``` -$ pip install -r requirements.txt -``` - -At this point you can now synthesize the CloudFormation template for this code. - -``` -$ cdk synth -``` - -To add additional dependencies, for example other CDK libraries, just add -them to your `setup.py` file and rerun the `pip install -r requirements.txt` -command. - -## Useful commands - - * `cdk ls` list all stacks in the app - * `cdk synth` emits the synthesized CloudFormation template - * `cdk deploy` deploy this stack to your default AWS account/region - * `cdk diff` compare deployed stack with current state - * `cdk docs` open CDK documentation - - ## Context parameters -There are multiple context parameters that you need to set before synthesizing or delpoying this CDK stack. You can specify a context variable either as part of an AWS CDK CLI command, or in `cdk.json`. -To create a command line context variable, use the __--context (-c) option__, as shown in the following example. - -``` -$ cdk cdk synth -c bucket_name=mybucket -``` - -To specify the same context variable and value in the cdk.json file, use the following code. - -``` -{ - "context": { - "bucket_name": "mybucket" - } -} -``` - -In this project, these are the following parameters to be set: - -* `topic_sql` -It is required for IoT Core rule creation to add a simplified SQL syntax to filter messages received on an MQTT topic and push the data elsewhere. -
__Format__: Enter an SQL statement using the following: ```SELECT FROM WHERE ```. For example: ```SELECT temperature FROM 'iot/topic' WHERE temperature > 50```. To learn more, see AWS IoT SQL Reference. - -* `opensearch_domain_name`    `` -The name of the Opensearch domain that will be created. -
__Format__: The name must start with a lowercase letter and must be between 3 and 28 characters. Valid characters are a-z (lowercase only), 0-9, and - (hyphen). - -* `opensearch_index_name`    `` -Before you can search data, you must index it. Indexing is the method by which search engines organize data for fast retrieval. The resulting structure is called, fittingly, an index. The name of the index that the IoT Core will use to send data to Opensearch is set via this parameter. -
__Format__: OpenSearch Service indexes have the following naming restrictions: - - * All letters must be lowercase. - - * Index names cannot begin with `_` or `-`. - - * Index names can't contain `spaces, commas, :, ", *, +, /, \, |, ?, #, >, or <`. - -* `opensearch_type_name`    `` -It resresents the type of the document that is going to be put under the index. Take the following example:
``` { - "_index" : "movies", - "_type" : "_doc", - "_id" : "1", - "_score" : 0.2876821, - "_source" : { - "director" : "Burton, Tim", - "genre" : [ - "Comedy", - "Sci-Fi" - ], - "year" : 1996, - "actor" : [ - "Jack Nicholson", - "Pierce Brosnan", - "Sarah Jessica Parker" - ], - "title" : "Mars Attacks!" - }``` - -* `cognito_user_pool_name`    `` -During the user pool creation process, you must specify a user pool name. This name can't be changed after the user pool has been created. -
__Format__: User pool names must be between one and 128 characters long. They can contain uppercase and lowercase letters (a-z, A-Z), numbers (0-9), and the following special characters: + = , . @ and -. - -* `cognito_user_pool_domain_name`    `` -The domain name for the domain that hosts the sign-up and sign-in pages for your application. -
__Format__: This string can include only lowercase letters, numbers, and hyphens. Don't use a hyphen for the first or last character. Use periods to separate subdomain names. It must be between 1 and 63 characters. - -* `cognito_identity_pool_name`    `` -Identity pools are used to store end user identities. To declare a new identity pool, you should provide a unique name. -
__Format__: It must be between 1 and 128 characters and follow such pattern: ```[\w\s+=,.@-]+ ``` - -* `cognito_user_username`    `` - Each user has a username attribute. Amazon Cognito automatically generates a user name for federated users. You must provide a username attribute to create a native user in the Amazon Cognito directory. After you create a user, you can't change the value of the username attribute. - - * `iot_to_opensearch_rule_name`    `` -The name of the IoT Core rule that is going to be created. -
__Format__: Should be an alphanumeric string that can also contain underscore (_) characters, but no spaces - - * `iot_to_opensearch_role_name`    `` -An IAM role should be created to grant AWS IoT access to Opensearch. This parameter is for setting the name of this role. -
__Format__: Enter a unique role name that contains alphanumeric characters, hyphens, and underscores. A role name can't contain any spaces. - -Enjoy! \ No newline at end of file diff --git a/Cloud_Templates/AWS_CDK/OpenSearchPattern/app.py b/Cloud_Templates/AWS_CDK/OpenSearchPattern/app.py deleted file mode 100644 index 1b7a79f..0000000 --- a/Cloud_Templates/AWS_CDK/OpenSearchPattern/app.py +++ /dev/null @@ -1,27 +0,0 @@ -import os - -import aws_cdk as cdk - -from open_search_pattern.open_search_pattern_stack import OpenSearchPatternStack - - -app = cdk.App() -OpenSearchPatternStack(app, "OpenSearchPatternStack", - # If you don't specify 'env', this stack will be environment-agnostic. - # Account/Region-dependent features and context lookups will not work, - # but a single synthesized template can be deployed anywhere. - - # Uncomment the next line to specialize this stack for the AWS Account - # and Region that are implied by the current CLI configuration. - - #env=cdk.Environment(account=os.getenv('CDK_DEFAULT_ACCOUNT'), region=os.getenv('CDK_DEFAULT_REGION')), - - # Uncomment the next line if you know exactly what Account and Region you - # want to deploy the stack to. */ - - #env=cdk.Environment(account='123456789012', region='us-east-1'), - - # For more information, see https://docs.aws.amazon.com/cdk/latest/guide/environments.html - ) - -app.synth() \ No newline at end of file diff --git a/Cloud_Templates/AWS_CDK/OpenSearchPattern/cdk.json b/Cloud_Templates/AWS_CDK/OpenSearchPattern/cdk.json deleted file mode 100644 index 7bc14e0..0000000 --- a/Cloud_Templates/AWS_CDK/OpenSearchPattern/cdk.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "app": "python3 app.py", - "watch": { - "include": [ - "**" - ], - "exclude": [ - "README.md", - "cdk*.json", - "requirements*.txt", - "source.bat", - "**/__init__.py", - "python/__pycache__", - "tests" - ] - }, - "context": { - "@aws-cdk/aws-apigateway:usagePlanKeyOrderInsensitiveId": true, - "@aws-cdk/core:stackRelativeExports": true, - "@aws-cdk/aws-rds:lowercaseDbIdentifier": true, - "@aws-cdk/aws-lambda:recognizeVersionProps": true, - "@aws-cdk/aws-lambda:recognizeLayerVersion": true, - "@aws-cdk/aws-cloudfront:defaultSecurityPolicyTLSv1.2_2021": true, - "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true, - "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true, - "@aws-cdk/core:checkSecretUsage": true, - "@aws-cdk/aws-iam:minimizePolicies": true, - "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true, - "@aws-cdk/core:validateSnapshotRemovalPolicy": true, - "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true, - "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true, - "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true, - "@aws-cdk/core:target-partitions": [ - "aws", - "aws-cn" - ], - "topic_sql": "SELECT * FROM 'Opensearch_demo'", - "opensearch_domain_name": "opensearch-demo-domain", - "opensearch_index_name": "iot", - "opensearch_type_name": "_doc", - "iot_to_opensearch_rule_name": "demo_to_opensearch_rule", - "iot_to_opensearch_role_name": "demo_iot_opensearch_role", - "cognito_user_pool_name" : "DemoUserPool", - "cognito_identity_pool_name": "DemoIdentityPool", - "cognito_user_pool_domain_name": "iot-demo-domain", - "cognito_user_username": "admin" - } -} diff --git a/Cloud_Templates/AWS_CDK/OpenSearchPattern/open_search_pattern/__init__.py b/Cloud_Templates/AWS_CDK/OpenSearchPattern/open_search_pattern/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/Cloud_Templates/AWS_CDK/OpenSearchPattern/open_search_pattern/open_search_pattern_stack.py b/Cloud_Templates/AWS_CDK/OpenSearchPattern/open_search_pattern/open_search_pattern_stack.py deleted file mode 100644 index b2bd2e0..0000000 --- a/Cloud_Templates/AWS_CDK/OpenSearchPattern/open_search_pattern/open_search_pattern_stack.py +++ /dev/null @@ -1,370 +0,0 @@ -from importlib import resources -from attr import attr -from aws_cdk import ( - Stack, - aws_iam as iam, - aws_iot as iot, - aws_logs as logs, - aws_opensearchservice as opensearch, - aws_cognito as cognito -) -from constructs import Construct -import aws_cdk as cdk -import datetime -import string -import sys -import secrets -import re - -sys.path.append('../') -from customExceptions import * - -class OpenSearchPatternStack(Stack): - - # Defining class variables - topic_sql = "" - opensearch_domain_name = "" - opensearch_index_name = "" - opensearch_type_name = "" - cognito_user_pool_name = "" - cognito_user_pool_domain_name = "" - cognito_identity_pool_name = "" - cognito_user_username = "" - iot_to_opensearch_rule_name = "" - iot_to_opensearch_role_name = "" - - def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: - super().__init__(scope, construct_id, **kwargs) - - # Get the context parameters - - # Required parameters for users to set in the CLI command or cdk.json - self.topic_sql = self.node.try_get_context("topic_sql") - - # Optional parameters for users to set in the CLI command or cdk.json - self.opensearch_domain_name = self.node.try_get_context("opensearch_domain_name") - self.opensearch_index_name = self.node.try_get_context("opensearch_index_name") - self.opensearch_type_name = self.node.try_get_context("opensearch_type_name") - self.cognito_user_pool_name = self.node.try_get_context("cognito_user_pool_name") - self.cognito_user_pool_domain_name = self.node.try_get_context("cognito_user_pool_domain_name") - self.cognito_identity_pool_name = self.node.try_get_context("cognito_identity_pool_name") - self.cognito_user_username = self.node.try_get_context("cognito_user_username") - self.iot_to_opensearch_rule_name = self.node.try_get_context("iot_to_opensearch_rule_name") - self.master_user_role_name = self.node.try_get_context("master_user_role_name") - - # Input Validation - self.performInputValidation() - - # Creating cognito identity pool - identityPool = cognito.CfnIdentityPool(self, self.cognito_identity_pool_name, allow_unauthenticated_identities=False) - identityPool.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) - - # Creating an IAM role as the master user for the fine-grained access control - masterUserRole = iam.CfnRole(self, self.master_user_role_name, assume_role_policy_document={ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "cognito-identity.amazonaws.com" - }, - "Action": "sts:AssumeRoleWithWebIdentity", - "Condition": { - "StringEquals": { - "cognito-identity.amazonaws.com:aud": identityPool.ref - }, - "ForAnyValue:StringLike": { - "cognito-identity.amazonaws.com:amr": "authenticated" - } - } - }, - { - "Effect": "Allow", - "Principal": { - "Service": "iot.amazonaws.com" - }, - "Action": "sts:AssumeRole" - } - ] - }) - masterUserRole.node.add_dependency(identityPool) - masterUserRole.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) - - # Creating cognito User pool and a domain for it - userPool = cognito.UserPool(self, self.cognito_user_pool_name) - userPool.add_domain(id=self.cognito_user_pool_domain_name, cognito_domain=cognito.CognitoDomainOptions( - domain_prefix=self.cognito_user_pool_domain_name)) - userPool.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) - - # Creating a UserPoolUser with a temporary password - masterUser = cdk.custom_resources.AwsCustomResource(self, "UserPoolUserCreation", - policy=cdk.custom_resources.AwsCustomResourcePolicy.from_statements([ - iam.PolicyStatement(effect=iam.Effect.ALLOW, actions=["cognito-idp:*"], resources=["*"]) - ]), - on_create=cdk.custom_resources.AwsSdkCall( - service='CognitoIdentityServiceProvider', - action='adminCreateUser', - parameters={ - "UserPoolId": userPool.user_pool_id, - "Username": self.cognito_user_username, - "TemporaryPassword": self.randomTemporaryPasswordGenerator(10) - }, - physical_resource_id=cdk.custom_resources.PhysicalResourceId.of('userpoolcreateid' + datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) - )) - - - # Creating a UserPoolGroup in the UserPool - user_pool_group = cognito.CfnUserPoolGroup(self, "master-user-group", group_name="master-user-group", - role_arn=masterUserRole.attr_arn, user_pool_id=userPool.user_pool_id) - user_pool_group.node.add_dependency(userPool) - user_pool_group.node.add_dependency(masterUserRole) - user_pool_group.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) - - # Attaching the user to the group - user_to_group_attachment = cognito.CfnUserPoolUserToGroupAttachment(self, "CDK_master_user_group_attachement", - group_name=user_pool_group.group_name, username=self.cognito_user_username, user_pool_id=userPool.user_pool_id) - user_to_group_attachment.node.add_dependency(user_pool_group) - user_to_group_attachment.node.add_dependency(userPool) - user_to_group_attachment.node.add_dependency(masterUser) - user_to_group_attachment.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) - - # Creating an IAM role to grant Opensearch access to Cognito - cognitoAccessForOpenSearchRole = iam.Role(self, "CDKCognitoAccessForOpenSearch", assumed_by=iam.ServicePrincipal("es.amazonaws.com")) - cognitoAccessForOpenSearchRole.add_managed_policy(iam.ManagedPolicy.from_aws_managed_policy_name("AmazonOpenSearchServiceCognitoAccess")) - cognitoAccessForOpenSearchRole.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) - - # Creating an Opensearch domain - domain = opensearch.Domain(self, self.opensearch_domain_name, - domain_name=self.opensearch_domain_name, - version=opensearch.EngineVersion.OPENSEARCH_1_2, - node_to_node_encryption=True, - encryption_at_rest=opensearch.EncryptionAtRestOptions( - enabled=True - ), - capacity=opensearch.CapacityConfig( - data_nodes=3, - data_node_instance_type="t3.small.search" - ), - enforce_https=True, - fine_grained_access_control=opensearch.AdvancedSecurityOptions( - master_user_arn=masterUserRole.attr_arn - ), - cognito_dashboards_auth=opensearch.CognitoOptions( - identity_pool_id=identityPool.ref, - user_pool_id=userPool.user_pool_id, - role=cognitoAccessForOpenSearchRole - )) - domain.add_access_policies(iam.PolicyStatement(effect=iam.Effect.ALLOW, resources=[domain.domain_arn+"/*"], actions=["es:ESHttp*"], principals=[iam.AnyPrincipal()])) - domain.node.add_dependency(identityPool) - domain.node.add_dependency(userPool) - domain.node.add_dependency(cognitoAccessForOpenSearchRole) - domain.node.add_dependency(masterUserRole) - domain.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) - - # Getting the Client ID of the opensearch client in UserPool - userPoolClients = cdk.custom_resources.AwsCustomResource(self, "ClientIdResource", - policy=cdk.custom_resources.AwsCustomResourcePolicy.from_sdk_calls( - resources=[userPool.user_pool_arn] - ), - on_create=cdk.custom_resources.AwsSdkCall( - service='CognitoIdentityServiceProvider', - action='listUserPoolClients', - parameters={"UserPoolId" : userPool.user_pool_id}, - physical_resource_id=cdk.custom_resources.PhysicalResourceId.of(f'ClientId-{self.cognito_user_pool_domain_name}') - )) - userPoolClients.node.add_dependency(domain) - clientID = userPoolClients.get_response_field('UserPoolClients.0.ClientId') - - # Changing Use default role to Choose role from token in Authentication Providers and For Role resolution, choose DENY. (Identity Pool Settigns) - roleAttachement = cognito.CfnIdentityPoolRoleAttachment(self, "CDKIdentityPoolAttachment", - identity_pool_id=identityPool.ref, - roles={}, - role_mappings={ - "role_mappings_key": cognito.CfnIdentityPoolRoleAttachment.RoleMappingProperty( - type='Token', - ambiguous_role_resolution='Deny', - identity_provider=f'cognito-idp.{self.region}.amazonaws.com/{userPool.user_pool_id}:{clientID}' - ) - }) - roleAttachement.node.add_dependency(domain) - roleAttachement.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) - - # Changing masterUserRole to also grant IoT access to Opensearch - masterUserRole.policies = [iam.CfnRole.PolicyProperty( - policy_document={ - "Version": "2012-10-17", - "Statement": [ - { - "Action": "iotanalytics:BatchPutMessage", - "Resource": f"arn:aws:iotanalytics:{self.region}:{self.account}:domain/{self.opensearch_domain_name}/*", - "Effect": "Allow" - } - ] - }, - policy_name="master_useriot_integration")] - - # Creating a cloud watch log group to capture any errors while sending the data through the IoT rule - log_group = logs.LogGroup(self, "iot_to_opensearch_log_group" , log_group_name="iot_to_opensearch_log_group", removal_policy=cdk.RemovalPolicy.DESTROY) - - # Creating the role to access the cloudwatch log group from IoT Core - iot_cloudwatch_log_role = iam.Role(self, "iot_cloudwatch_error_log_role", assumed_by=iam.ServicePrincipal("iot.amazonaws.com")) - iot_cloudwatch_log_role.add_to_policy(iam.PolicyStatement(effect=iam.Effect.ALLOW, resources=[log_group.log_group_arn], actions=["logs:CreateLogStream","logs:DescribeLogStreams","logs:PutLogEvents"])) - iot_cloudwatch_log_role.node.add_dependency(log_group) - iot_cloudwatch_log_role.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) - - # Creating the IoT Core Rule + Setting the error action to log in the cloud watch log group - topic_rule = iot.CfnTopicRule(self, self.iot_to_opensearch_rule_name, topic_rule_payload=iot.CfnTopicRule.TopicRulePayloadProperty( - actions=[iot.CfnTopicRule.ActionProperty( open_search=iot.CfnTopicRule.OpenSearchActionProperty( - endpoint= "https://" + domain.domain_endpoint, - id="${newuuid()}", - index=self.opensearch_index_name, - role_arn=masterUserRole.attr_arn, - type=self.opensearch_type_name - ) - )], sql=self.topic_sql, - error_action = iot.CfnTopicRule.ActionProperty( - cloudwatch_logs=iot.CfnTopicRule.CloudwatchLogsActionProperty( - log_group_name=log_group.log_group_name, - role_arn=iot_cloudwatch_log_role.role_arn - )) - )) - topic_rule.node.add_dependency(domain) - topic_rule.node.add_dependency(masterUserRole) - topic_rule.node.add_dependency(log_group) - topic_rule.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) - - def randomTemporaryPasswordGenerator(self, length): - - source = string.ascii_letters + string.digits + string.punctuation - password = secrets.choice(string.ascii_lowercase) - password += secrets.choice(string.ascii_uppercase) - password += secrets.choice(string.digits) - password += secrets.choice(string.punctuation) - - for i in range(length-4): - password += secrets.choice(source) - - char_list = list(password) - secrets.SystemRandom().shuffle(char_list) - password = ''.join(char_list) - - print(f"The temporary password created for the Cognito user is {password}\nPlease keep this password for the first login and then change it to a secure one of your own.") - return password - - def performInputValidation(self): - self.validateTopicSQL(self.topic_sql) - self.validateOpensearchDomainName(self.opensearch_domain_name) - self.validateOpensearchIndexName(self.opensearch_index_name) - self.validateOpensearchTypeName(self.opensearch_type_name) - self.validateCognitoUserPoolName(self.cognito_user_pool_name) - self.validateCognitoUserPoolDomainName(self.cognito_user_pool_domain_name) - self.validateCognitoIdentityPoolName(self.cognito_identity_pool_name) - self.validateCognitoUserUsername(self.cognito_user_username) - self.validateIoTRuleName(self.iot_to_opensearch_rule_name) - self.validateIAMRoleName(self.master_user_role_name) - - def validateTopicSQL(self, input): - if not input: - raise NoSQL - elif type(input) != str: - raise WrongFormattedInput("The input sql statement does not have a right format. Please refer to README.md for more information.") - return - - def validateOpensearchDomainName(self, input): - if not input: - self.opensearch_domain_name = "opensearch-demo-domain" - elif type(input) != str: - raise WrongFormattedInput("The Opensearch domain name should be of a string format.") - elif len(input) < 3 or len(input) > 28: - raise WrongLengthForInput("The Opensearch domain name must be between 3 and 28 characters.") - elif not input[0].islower(): - raise WrongFormattedInput("The Opensearch domain name must start with a lowercase letter.") - elif not re.match(r'^[a-z0-9-]+$', input): - raise WrongFormattedInput("Valid characters for Opensearch domain name are a-z (lowercase only), 0-9, and - (hyphen).") - else: - return - - def validateOpensearchIndexName(self, input): - not_allowed_characters = [' ', ',', ':', '\"', '*', '+', '/', '\\', '|', '?', '#', '>' , '<'] - if not input: - self.opensearch_index_name = "iot" - elif type(input) != str: - raise WrongFormattedInput("The Opensearch index name should be of a string format.") - elif not input.islower(): - raise WrongFormattedInput("All letters of the Opensearch index name must be lowercase.") - elif input[0] == "_" or input[0] == "-": - raise WrongFormattedInput("Opensearch index names cannot begin with `_` or `-`.") - elif any(c in input for c in not_allowed_characters): - raise WrongFormattedInput("Opensearch index names cannot contain `spaces, commas, :, \", *, +, /, \\, |, ?, #, >, or <`.") - else: - return - - def validateOpensearchTypeName(self, input): - if not input: - self.opensearch_type_name = "_doc" - else: - return - - def validateCognitoUserPoolName(self, input): - if not input: - self.cognito_user_pool_name = "DemoUserPool" - elif type(input) != str: - raise WrongFormattedInput("The Cognito UserPool name should be of a string format.") - elif len(input) > 128: - raise WrongLengthForInput("The Cognito UserPool name must be between one and 128 characters long.") - elif not re.match(r'^[a-zA-Z0-9-+=,@\.]+$', input): - raise WrongFormattedInput("The Cognito UserPool name can contain uppercase and lowercase letters, numbers, and the following special characters: + = , . @ and -") - else: - return - - def validateCognitoUserPoolDomainName(self, input): - if not input: - self.cognito_user_pool_domain_name = "iot-demo-domain" - elif type(input) != str: - raise WrongFormattedInput("The Cognito UserPool domain name should be of a string format.") - elif len(input) > 63: - raise WrongLengthForInput("The Cognito UserPool domain name must be between 1 and 63 characters long.") - elif not re.match(r'^[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?$', input): - raise WrongFormattedInput("The Cognito UserPool domain name can include only lowercase letters, numbers, and hyphens. Don't use a hyphen for the first or last character.") - else: - return - - def validateCognitoIdentityPoolName(self, input): - if not input: - self.cognito_identity_pool_name = "DemoIdentityPool" - elif type(input) != str: - raise WrongFormattedInput("The Cognito IdentityPool name should be of a string format.") - elif len(input) > 128: - raise WrongLengthForInput("The Cognito IdentityPool name must be between 1 and 128 characters long.") - elif not re.match(r'^[\w\s+=,.@-]+$', input): - raise WrongFormattedInput("Wrong format for the Cognito IdentityPool name. Please refer to README.md for more information.") - else: - return - - def validateCognitoUserUsername(self, input): - if not input: - self.cognito_user_username = "admin" - else: - return - - def validateIoTRuleName(self, input): - if not input: - self.iot_to_opensearch_rule_name = "demo_iot_opensearch_rule" - elif type(input) != str: - raise WrongFormattedInput("The provided input for IoT topic rule name is not of type string.") - elif not re.match(r'^[a-zA-Z0-9-_\.]+$', input): - raise WrongFormattedInput("The IoT topic rule name should be an alphanumeric string that can also contain underscore (_) characters, but no spaces.") - else: - return - - def validateIAMRoleName(self, input): - if not input: - self.master_user_role_name = "demo_iot_opensearch_role" - elif type(input) != str: - raise WrongFormattedInput("The provided input for the IAM role name is not of type string.") - elif len(input) > 64: - raise WrongLengthForInput("The length of the IAM role name string should not exceed 64 characters.") - elif not re.match(r'^[a-zA-Z0-9+=,@-_\.]+$', input): - raise WrongFormattedInput("The IAM role name should be an alphanumeric string that can also contain '+=,.@-_' characters.") - else: - return \ No newline at end of file diff --git a/Cloud_Templates/AWS_CDK/OpenSearchPattern/requirements-dev.txt b/Cloud_Templates/AWS_CDK/OpenSearchPattern/requirements-dev.txt deleted file mode 100644 index 9270945..0000000 --- a/Cloud_Templates/AWS_CDK/OpenSearchPattern/requirements-dev.txt +++ /dev/null @@ -1 +0,0 @@ -pytest==6.2.5 diff --git a/Cloud_Templates/AWS_CDK/OpenSearchPattern/requirements.txt b/Cloud_Templates/AWS_CDK/OpenSearchPattern/requirements.txt deleted file mode 100644 index 0822bbe..0000000 --- a/Cloud_Templates/AWS_CDK/OpenSearchPattern/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -aws-cdk-lib==2.37.1 -constructs>=10.0.0,<11.0.0 diff --git a/Cloud_Templates/AWS_CDK/OpenSearchPattern/source.bat b/Cloud_Templates/AWS_CDK/OpenSearchPattern/source.bat deleted file mode 100644 index 9e1a834..0000000 --- a/Cloud_Templates/AWS_CDK/OpenSearchPattern/source.bat +++ /dev/null @@ -1,13 +0,0 @@ -@echo off - -rem The sole purpose of this script is to make the command -rem -rem source .venv/bin/activate -rem -rem (which activates a Python virtualenv on Linux or Mac OS X) work on Windows. -rem On Windows, this command just runs this batch file (the argument is ignored). -rem -rem Now we don't need to document a Windows command for activating a virtualenv. - -echo Executing .venv\Scripts\activate.bat for you -.venv\Scripts\activate.bat diff --git a/Cloud_Templates/AWS_CDK/OpenSearchPattern/tests/__init__.py b/Cloud_Templates/AWS_CDK/OpenSearchPattern/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/Cloud_Templates/AWS_CDK/OpenSearchPattern/tests/unit/__init__.py b/Cloud_Templates/AWS_CDK/OpenSearchPattern/tests/unit/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/Cloud_Templates/AWS_CDK/OpenSearchPattern/tests/unit/test_open_search_pattern_stack.py b/Cloud_Templates/AWS_CDK/OpenSearchPattern/tests/unit/test_open_search_pattern_stack.py deleted file mode 100644 index 77213aa..0000000 --- a/Cloud_Templates/AWS_CDK/OpenSearchPattern/tests/unit/test_open_search_pattern_stack.py +++ /dev/null @@ -1,798 +0,0 @@ -import aws_cdk as core -import aws_cdk.assertions as assertions -from aws_cdk.assertions import Match -import pytest - -from open_search_pattern.open_search_pattern_stack import OpenSearchPatternStack - -# Setting the context for the app -app = core.App(context={ - "topic_sql": "SELECT * FROM 'Opensearch_demo'", - "opensearch_domain_name": "cdk-opensearch-domain", - "opensearch_index_name": "measurements", - "opensearch_type_name": "_doc", - "iot_to_opensearch_rule_name": "cdk_iot_opensearchFineGrained_rule", - "master_user_role_name": "masterUserRole", - "cognito_user_pool_name" : "CDKUserPool", - "cognito_identity_pool_name": "CDKIdentityPool", - "cognito_user_pool_domain_name": "cdk-iot-domain", - "cognito_user_username": "admin" -}) - -stack = OpenSearchPatternStack(app, "open-search-pattern") -template = assertions.Template.from_stack(stack) - -# Defining Capture objects for obtaining values in tests -userPool_ref_capture = assertions.Capture() -userpooluser_lambda_ref_capture = assertions.Capture() -userPool_group_name = assertions.Capture() -userPool_group_ref_capture = assertions.Capture() -userPool_domain_ref_capture = assertions.Capture() -fineGrainedRole_ref_capture = assertions.Capture() -identity_pool_ref_capture = assertions.Capture() -cognitoAccessForOpenSearch_role_ref_capture = assertions.Capture() -opensearch_domain_ref_capture = assertions.Capture() -opensearch_domain_access_policy_ref_capture = assertions.Capture() -custom_clientIdResource_ref_capture = assertions.Capture() -log_group_ref_capture = assertions.Capture() -log_group_role_ref_capture = assertions.Capture() -log_group_policy_ref_capture = assertions.Capture() -userPoolUserCreation_ref_capture = assertions.Capture() -userPoolUserCreationCustomResourcePolicy_ref_capture = assertions.Capture() -masterUserRole_ref_capture = assertions.Capture() - -# Testing the resources' creation and properties - -def test_identity_pool_creation(): - template.has_resource("AWS::Cognito::IdentityPool", {"DeletionPolicy":"Delete", "UpdateReplacePolicy":"Delete"}) - template.resource_count_is("AWS::Cognito::IdentityPool",1) - -def test_identity_pool_properties(): - template.has_resource_properties("AWS::Cognito::IdentityPool", { - "AllowUnauthenticatedIdentities": False - }) - -def test_masterUser_IAM_role_properties(): - template.has_resource_properties("AWS::IAM::Role", { - "AssumeRolePolicyDocument": { - "Version": Match.any_value(), - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "cognito-identity.amazonaws.com" - }, - "Action": "sts:AssumeRoleWithWebIdentity", - "Condition": { - "StringEquals": { - "cognito-identity.amazonaws.com:aud": { - "Ref": app.node.try_get_context("cognito_identity_pool_name") - } - }, - "ForAnyValue:StringLike": { - "cognito-identity.amazonaws.com:amr": "authenticated" - } - } - }, - { - "Effect": "Allow", - "Principal": { - "Service": "iot.amazonaws.com" - }, - "Action": "sts:AssumeRole" - } - ] - }, - "Policies": [{ - "PolicyDocument": { - "Version": Match.any_value(), - "Statement": [ - { - "Action": "iotanalytics:BatchPutMessage", - "Resource": { - "Fn::Join": [ - "", - [ - "arn:aws:iotanalytics:", - { - "Ref": "AWS::Region" - }, - ":", - { - "Ref": "AWS::AccountId" - }, - f':domain/{app.node.try_get_context("opensearch_domain_name")}/*' - ] - ] - }, - "Effect": "Allow" - } - ] - }, - "PolicyName": Match.any_value() - }] - }) - -def test_user_pool_creation(): - template.has_resource("AWS::Cognito::UserPool", {"DeletionPolicy":"Delete", "UpdateReplacePolicy":"Delete"}) - template.resource_count_is("AWS::Cognito::UserPool",1) - -def test_user_pool_properties(): - template.has_resource_properties("AWS::Cognito::UserPool", { - "AccountRecoverySetting": { - "RecoveryMechanisms": [ - { - "Name": "verified_phone_number", - "Priority": 1 - }, - { - "Name": "verified_email", - "Priority": 2 - } - ] - }, - "AdminCreateUserConfig": { - "AllowAdminCreateUserOnly": True - } - }) - -def test_user_pool_domain_creation(): - template.has_resource("AWS::Cognito::UserPoolDomain", {}) - template.resource_count_is("AWS::Cognito::UserPoolDomain",1) - -def test_user_pool_domain_properties(): - template.has_resource_properties("AWS::Cognito::UserPoolDomain", { - "Domain": app.node.try_get_context("cognito_user_pool_domain_name"), - "UserPoolId": { - "Ref": userPool_ref_capture - } - }) - -def test_user_pool_user_creation(): - template.has_resource("Custom::AWS", {"DeletionPolicy":"Delete", "UpdateReplacePolicy":"Delete"}) - -def test_user_pool_user_properties(): - template.has_resource_properties("Custom::AWS", { - "ServiceToken": { - "Fn::GetAtt": [ - userpooluser_lambda_ref_capture, - "Arn" - ] - }, - "Create": Match.any_value(), - "InstallLatestAwsSdk": True - }) - -def test_user_pool_user_group_creation(): - template.has_resource("AWS::Cognito::UserPoolGroup", {"DeletionPolicy":"Delete", "UpdateReplacePolicy":"Delete"}) - template.resource_count_is("AWS::Cognito::UserPoolGroup", 1) - -def test_user_pool_user_group_properties(): - template.has_resource_properties("AWS::Cognito::UserPoolGroup", { - "UserPoolId": { - "Ref": userPool_ref_capture.as_string() - }, - "GroupName": userPool_group_name, - "RoleArn": { - "Fn::GetAtt": [ - app.node.try_get_context("master_user_role_name"), - "Arn" - ] - } - }) - -def test_userPoolUserToGroupAttachment_creation(): - template.has_resource("AWS::Cognito::UserPoolUserToGroupAttachment", {"DeletionPolicy":"Delete", "UpdateReplacePolicy":"Delete"}) - template.resource_count_is("AWS::Cognito::UserPoolUserToGroupAttachment", 1) - -def test_userPoolUserToGroupAttachment_properties(): - template.has_resource_properties("AWS::Cognito::UserPoolUserToGroupAttachment", { - "GroupName": userPool_group_name.as_string(), - "Username": app.node.try_get_context("cognito_user_username"), - "UserPoolId": { - "Ref": userPool_ref_capture.as_string() - } - }) - -def test_CognitoAccessForOpenSearch_role_properties(): - template.has_resource_properties("AWS::IAM::Role", { - "AssumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "es.amazonaws.com" - } - }], - "Version": Match.any_value() - }, - "ManagedPolicyArns": [ - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":iam::aws:policy/AmazonOpenSearchServiceCognitoAccess" - ]] - }] - }) - -def test_opensearch_domain_creation(): - template.has_resource("AWS::OpenSearchService::Domain", {"DeletionPolicy":"Delete", "UpdateReplacePolicy":"Delete"}) - template.resource_count_is("AWS::OpenSearchService::Domain",1) - -def test_opensearch_domain_properties(): - template.has_resource_properties("AWS::OpenSearchService::Domain", { - "AdvancedSecurityOptions": { - "Enabled": True, - "InternalUserDatabaseEnabled": False, - "MasterUserOptions": { - "MasterUserARN": { - "Fn::GetAtt": [ - fineGrainedRole_ref_capture, - "Arn" - ] - } - } - }, - "ClusterConfig": { - "DedicatedMasterEnabled": False, - "InstanceCount": Match.any_value(), - "InstanceType": Match.any_value(), - "ZoneAwarenessEnabled": False - }, - "CognitoOptions": { - "Enabled": True, - "IdentityPoolId": { - "Ref": identity_pool_ref_capture - }, - "RoleArn": { - "Fn::GetAtt": [ - cognitoAccessForOpenSearch_role_ref_capture, - "Arn" - ] - }, - "UserPoolId": { - "Ref": userPool_ref_capture.as_string() - } - }, - "DomainEndpointOptions": { - "EnforceHTTPS": True, - "TLSSecurityPolicy": Match.any_value() - }, - "DomainName": app.node.try_get_context("opensearch_domain_name"), - "EBSOptions": { - "EBSEnabled": True, - "VolumeSize": Match.any_value(), - "VolumeType": Match.any_value() - }, - "EncryptionAtRestOptions": { - "Enabled": True - }, - "EngineVersion": Match.any_value(), - "LogPublishingOptions": {}, - "NodeToNodeEncryptionOptions": { - "Enabled": True - } - }) - -def test_opensearch_access_policy_properties(): - template.has_resource_properties("AWS::IAM::Policy", { - "PolicyDocument": { - "Statement": [ - { - "Action": "es:UpdateDomainConfig", - "Effect": "Allow", - "Resource": { - "Fn::GetAtt": [ - opensearch_domain_ref_capture, - "Arn" - ] - } - } - ], - "Version": Match.any_value() - }, - "PolicyName": opensearch_domain_access_policy_ref_capture, - "Roles": [ - { - "Ref": Match.any_value() - } - ] - }) - -def test_IdentityPoolRoleAttachment_creation(): - template.has_resource("AWS::Cognito::IdentityPoolRoleAttachment", {"DeletionPolicy":"Delete", "UpdateReplacePolicy":"Delete"}) - template.resource_count_is("AWS::Cognito::IdentityPoolRoleAttachment", 1) - -def test_IdentityPoolRoleAttachment_properties(): - template.has_resource_properties("AWS::Cognito::IdentityPoolRoleAttachment", { - "IdentityPoolId": { - "Ref": identity_pool_ref_capture.as_string() - }, - "RoleMappings": { - "role_mappings_key": { - "AmbiguousRoleResolution": "Deny", - "IdentityProvider": { - "Fn::Join": [ - "", - [ - "cognito-idp.", - { - "Ref": "AWS::Region" - }, - ".amazonaws.com/", - { - "Ref": userPool_ref_capture.as_string() - }, - ":", - { - "Fn::GetAtt": [ - custom_clientIdResource_ref_capture, - "UserPoolClients.0.ClientId" - ] - } - ]] - }, - "Type": "Token" - } - }, - "Roles": {} - }) - -def test_log_group_creation(): - template.has_resource("AWS::Logs::LogGroup", {"DeletionPolicy":"Delete", "UpdateReplacePolicy":"Delete"}) - template.resource_count_is("AWS::Logs::LogGroup",1) - -def test_log_group_properties(): - template.has_resource_properties("AWS::Logs::LogGroup", { - "LogGroupName": "iot_to_opensearch_log_group", - "RetentionInDays": Match.any_value() - }) - -def test_log_group_role_properties(): - template.has_resource_properties("AWS::IAM::Role", { - "AssumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "iot.amazonaws.com" - } - } - ], - "Version": Match.any_value() - } - }) - -def test_log_group_policy_properties(): - template.has_resource_properties("AWS::IAM::Policy", { - "PolicyDocument": { - "Statement": [ - { - "Action": [ - "logs:CreateLogStream", - "logs:DescribeLogStreams", - "logs:PutLogEvents" - ], - "Effect": "Allow", - "Resource": { - "Fn::GetAtt": [ - log_group_ref_capture, - "Arn" - ] - } - } - ], - "Version": Match.any_value() - }, - "PolicyName": log_group_policy_ref_capture, - "Roles": [ - { - "Ref": log_group_role_ref_capture - } - ] - }) - -def test_iot_rule_creation(): - template.has_resource("AWS::IoT::TopicRule", {"DeletionPolicy":"Delete", "UpdateReplacePolicy":"Delete"}) - template.resource_count_is("AWS::IoT::TopicRule",1) - -def test_iot_rule_properties(): - template.has_resource_properties("AWS::IoT::TopicRule", { - "TopicRulePayload": { - "Actions": [ - { - "OpenSearch": { - "Endpoint": { - "Fn::Join": [ - "", - [ - "https://", - { - "Fn::GetAtt": [ - opensearch_domain_ref_capture.as_string(), - "DomainEndpoint" - ] - } - ] - ] - }, - "Id": "${newuuid()}", - "Index": app.node.try_get_context("opensearch_index_name"), - "RoleArn": { - "Fn::GetAtt": [ - fineGrainedRole_ref_capture.as_string(), - "Arn" - ] - }, - "Type": app.node.try_get_context("opensearch_type_name") - } - } - ], - "ErrorAction": { - "CloudwatchLogs": { - "LogGroupName": { - "Ref": log_group_ref_capture.as_string() - }, - "RoleArn": { - "Fn::GetAtt": [ - log_group_role_ref_capture.as_string(), - "Arn" - ] - } - } - }, - "Sql": app.node.try_get_context("topic_sql") - } - }) - -# Testing dependencies between the resources - -def test_masterUser_role_dependencies(): - template.has_resource("AWS::IAM::Role", { - "DependsOn": [ - identity_pool_ref_capture.as_string() - ] - }) - -def test_userPoolUserToGroupAttachment_dependencies(): - template.has_resource("AWS::Cognito::UserPoolUserToGroupAttachment", { - "DependsOn": [ - userPool_domain_ref_capture, - userPool_ref_capture.as_string(), - userPool_group_ref_capture, - userPoolUserCreationCustomResourcePolicy_ref_capture, - userPoolUserCreation_ref_capture - ] - }) - -def test_opensearch_domain_dependencies(): - template.has_resource("AWS::OpenSearchService::Domain", { - "DependsOn": [ - cognitoAccessForOpenSearch_role_ref_capture.as_string(), - identity_pool_ref_capture.as_string(), - userPool_domain_ref_capture.as_string(), - userPool_ref_capture.as_string(), - masterUserRole_ref_capture - ] - }) - -# Testing input validation process - -def test_no_sql(): - test_app = core.App(context= { - "opensearch_domain_name": "cdk-opensearch-domain", - "opensearch_index_name": "measurements", - "opensearch_type_name": "_doc", - "iot_to_opensearch_rule_name": "cdk_iot_opensearchFineGrained_rule", - "master_user_role_name": "masterUserRole", - "cognito_user_pool_name" : "CDKUserPool", - "cognito_identity_pool_name": "CDKIdentityPool", - "cognito_user_pool_domain_name": "cdk-iot-domain", - "cognito_user_username": "admin" - }) - with pytest.raises(Exception, match=r"No sql statemtnt .*"): - stack = OpenSearchPatternStack(test_app, "open-search-pattern") - template = assertions.Template.from_stack(stack) - -def test_wrong_sql(): - test_app = core.App(context= { - "topic_sql": ["SELECT * FROM 'Opensearch_demo'"], - "opensearch_domain_name": "cdk-opensearch-domain", - "opensearch_index_name": "measurements", - "opensearch_type_name": "_doc", - "iot_to_opensearch_rule_name": "cdk_iot_opensearchFineGrained_rule", - "master_user_role_name": "masterUserRole", - "cognito_user_pool_name" : "CDKUserPool", - "cognito_identity_pool_name": "CDKIdentityPool", - "cognito_user_pool_domain_name": "cdk-iot-domain", - "cognito_user_username": "admin" - }) - with pytest.raises(Exception, match=r"The input sql statement does not have a right format. .*"): - stack = OpenSearchPatternStack(test_app, "open-search-pattern") - template = assertions.Template.from_stack(stack) - -def test_wrong_opensearch_domain(): - test_app = core.App(context= { - "topic_sql": "SELECT * FROM 'Opensearch_demo'", - "opensearch_domain_name": ["cdk-opensearch-domain"], - "opensearch_index_name": "measurements", - "opensearch_type_name": "_doc", - "iot_to_opensearch_rule_name": "cdk_iot_opensearchFineGrained_rule", - "master_user_role_name": "masterUserRole", - "cognito_user_pool_name" : "CDKUserPool", - "cognito_identity_pool_name": "CDKIdentityPool", - "cognito_user_pool_domain_name": "cdk-iot-domain", - "cognito_user_username": "admin" - }) - with pytest.raises(Exception, match=r"The Opensearch domain name should be of a string format."): - stack = OpenSearchPatternStack(test_app, "open-search-pattern") - template = assertions.Template.from_stack(stack) - - test_app = core.App(context= { - "topic_sql": "SELECT * FROM 'Opensearch_demo'", - "opensearch_domain_name": "x" * 30, - "opensearch_index_name": "measurements", - "opensearch_type_name": "_doc", - "iot_to_opensearch_rule_name": "cdk_iot_opensearchFineGrained_rule", - "master_user_role_name": "masterUserRole", - "cognito_user_pool_name" : "CDKUserPool", - "cognito_identity_pool_name": "CDKIdentityPool", - "cognito_user_pool_domain_name": "cdk-iot-domain", - "cognito_user_username": "admin" - }) - with pytest.raises(Exception, match=r"The Opensearch domain name must be between 3 and 28 characters."): - stack = OpenSearchPatternStack(test_app, "open-search-pattern") - template = assertions.Template.from_stack(stack) - - test_app = core.App(context= { - "topic_sql": "SELECT * FROM 'Opensearch_demo'", - "opensearch_domain_name": "Domain", - "opensearch_index_name": "measurements", - "opensearch_type_name": "_doc", - "iot_to_opensearch_rule_name": "cdk_iot_opensearchFineGrained_rule", - "master_user_role_name": "masterUserRole", - "cognito_user_pool_name" : "CDKUserPool", - "cognito_identity_pool_name": "CDKIdentityPool", - "cognito_user_pool_domain_name": "cdk-iot-domain", - "cognito_user_username": "admin" - }) - with pytest.raises(Exception, match=r"The Opensearch domain name must start with a lowercase letter."): - stack = OpenSearchPatternStack(test_app, "open-search-pattern") - template = assertions.Template.from_stack(stack) - - test_app = core.App(context= { - "topic_sql": "SELECT * FROM 'Opensearch_demo'", - "opensearch_domain_name": "domain_iot", - "opensearch_index_name": "measurements", - "opensearch_type_name": "_doc", - "iot_to_opensearch_rule_name": "cdk_iot_opensearchFineGrained_rule", - "master_user_role_name": "masterUserRole", - "cognito_user_pool_name" : "CDKUserPool", - "cognito_identity_pool_name": "CDKIdentityPool", - "cognito_user_pool_domain_name": "cdk-iot-domain", - "cognito_user_username": "admin" - }) - with pytest.raises(Exception, match=r"Valid characters for Opensearch domain .*"): - stack = OpenSearchPatternStack(test_app, "open-search-pattern") - template = assertions.Template.from_stack(stack) - -def test_wrong_opensearch_index(): - test_app = core.App(context= { - "topic_sql": "SELECT * FROM 'Opensearch_demo'", - "opensearch_domain_name": "cdk-opensearch-domain", - "opensearch_index_name": "Index", - "opensearch_type_name": "_doc", - "iot_to_opensearch_rule_name": "cdk_iot_opensearchFineGrained_rule", - "master_user_role_name": "masterUserRole", - "cognito_user_pool_name" : "CDKUserPool", - "cognito_identity_pool_name": "CDKIdentityPool", - "cognito_user_pool_domain_name": "cdk-iot-domain", - "cognito_user_username": "admin" - }) - with pytest.raises(Exception, match=r"All letters of the Opensearch index name must be lowercase."): - stack = OpenSearchPatternStack(test_app, "open-search-pattern") - template = assertions.Template.from_stack(stack) - - test_app = core.App(context= { - "topic_sql": "SELECT * FROM 'Opensearch_demo'", - "opensearch_domain_name": "cdk-opensearch-domain", - "opensearch_index_name": "_myindex", - "opensearch_type_name": "_doc", - "iot_to_opensearch_rule_name": "cdk_iot_opensearchFineGrained_rule", - "master_user_role_name": "masterUserRole", - "cognito_user_pool_name" : "CDKUserPool", - "cognito_identity_pool_name": "CDKIdentityPool", - "cognito_user_pool_domain_name": "cdk-iot-domain", - "cognito_user_username": "admin" - }) - with pytest.raises(Exception, match=r"Opensearch index names cannot begin with `_` or `-`."): - stack = OpenSearchPatternStack(test_app, "open-search-pattern") - template = assertions.Template.from_stack(stack) - - test_app = core.App(context= { - "topic_sql": "SELECT * FROM 'Opensearch_demo'", - "opensearch_domain_name": "cdk-opensearch-domain", - "opensearch_index_name": "my\\index", - "opensearch_type_name": "_doc", - "iot_to_opensearch_rule_name": "cdk_iot_opensearchFineGrained_rule", - "master_user_role_name": "masterUserRole", - "cognito_user_pool_name" : "CDKUserPool", - "cognito_identity_pool_name": "CDKIdentityPool", - "cognito_user_pool_domain_name": "cdk-iot-domain", - "cognito_user_username": "admin" - }) - with pytest.raises(Exception, match=r"Opensearch index names cannot contain .*"): - stack = OpenSearchPatternStack(test_app, "open-search-pattern") - template = assertions.Template.from_stack(stack) - -def test_wrong_userPool_domain_name(): - test_app = core.App(context= { - "topic_sql": "SELECT * FROM 'Opensearch_demo'", - "opensearch_domain_name": "cdk-opensearch-domain", - "opensearch_index_name": "index", - "opensearch_type_name": "_doc", - "iot_to_opensearch_rule_name": "cdk_iot_opensearchFineGrained_rule", - "master_user_role_name": "masterUserRole", - "cognito_user_pool_name" : "CDKUserPool", - "cognito_identity_pool_name": "CDKIdentityPool", - "cognito_user_pool_domain_name": "X" * 64, - "cognito_user_username": "admin" - }) - with pytest.raises(Exception, match=r"The Cognito UserPool domain name must be between 1 and 63 characters long."): - stack = OpenSearchPatternStack(test_app, "open-search-pattern") - template = assertions.Template.from_stack(stack) - - test_app = core.App(context= { - "topic_sql": "SELECT * FROM 'Opensearch_demo'", - "opensearch_domain_name": "cdk-opensearch-domain", - "opensearch_index_name": "index", - "opensearch_type_name": "_doc", - "iot_to_opensearch_rule_name": "cdk_iot_opensearchFineGrained_rule", - "master_user_role_name": "masterUserRole", - "cognito_user_pool_name" : "CDKUserPool", - "cognito_identity_pool_name": "CDKIdentityPool", - "cognito_user_pool_domain_name": "-cdk-iot-domain", - "cognito_user_username": "admin" - }) - with pytest.raises(Exception, match=r"The Cognito UserPool domain name can include .*"): - stack = OpenSearchPatternStack(test_app, "open-search-pattern") - template = assertions.Template.from_stack(stack) - - test_app = core.App(context= { - "topic_sql": "SELECT * FROM 'Opensearch_demo'", - "opensearch_domain_name": "cdk-opensearch-domain", - "opensearch_index_name": "index", - "opensearch_type_name": "_doc", - "iot_to_opensearch_rule_name": "cdk_iot_opensearchFineGrained_rule", - "master_user_role_name": "masterUserRole", - "cognito_user_pool_name" : "CDKUserPool", - "cognito_identity_pool_name": "CDKIdentityPool", - "cognito_user_pool_domain_name": "Cdk-iot-domain", - "cognito_user_username": "admin" - }) - with pytest.raises(Exception, match=r"The Cognito UserPool domain name can include .*"): - stack = OpenSearchPatternStack(test_app, "open-search-pattern") - template = assertions.Template.from_stack(stack) - -def test_wrong_userPool_name(): - test_app = core.App(context= { - "topic_sql": "SELECT * FROM 'Opensearch_demo'", - "opensearch_domain_name": "cdk-opensearch-domain", - "opensearch_index_name": "index", - "opensearch_type_name": "_doc", - "iot_to_opensearch_rule_name": "cdk_iot_opensearchFineGrained_rule", - "master_user_role_name": "masterUserRole", - "cognito_user_pool_name" : "c" * 129, - "cognito_identity_pool_name": "CDKIdentityPool", - "cognito_user_pool_domain_name": "cdk-iot-domain", - "cognito_user_username": "admin" - }) - with pytest.raises(Exception, match=r"The Cognito UserPool name must be between one and 128 characters long."): - stack = OpenSearchPatternStack(test_app, "open-search-pattern") - template = assertions.Template.from_stack(stack) - - test_app = core.App(context= { - "topic_sql": "SELECT * FROM 'Opensearch_demo'", - "opensearch_domain_name": "cdk-opensearch-domain", - "opensearch_index_name": "index", - "opensearch_type_name": "_doc", - "iot_to_opensearch_rule_name": "cdk_iot_opensearchFineGrained_rule", - "master_user_role_name": "masterUserRole", - "cognito_user_pool_name" : "CDK#UserPool", - "cognito_identity_pool_name": "CDKIdentityPool", - "cognito_user_pool_domain_name": "cdk-iot-domain", - "cognito_user_username": "admin" - }) - with pytest.raises(Exception, match=r"The Cognito UserPool name can contain .*"): - stack = OpenSearchPatternStack(test_app, "open-search-pattern") - template = assertions.Template.from_stack(stack) - -def test_wrong_identityPool_name(): - test_app = core.App(context= { - "topic_sql": "SELECT * FROM 'Opensearch_demo'", - "opensearch_domain_name": "cdk-opensearch-domain", - "opensearch_index_name": "index", - "opensearch_type_name": "_doc", - "iot_to_opensearch_rule_name": "cdk_iot_opensearchFineGrained_rule", - "master_user_role_name": "masterUserRole", - "cognito_user_pool_name" : "CDKUserPool", - "cognito_identity_pool_name": "c" * 129, - "cognito_user_pool_domain_name": "cdk-iot-domain", - "cognito_user_username": "admin" - }) - with pytest.raises(Exception, match=r"The Cognito IdentityPool name must be between 1 and 128 characters long."): - stack = OpenSearchPatternStack(test_app, "open-search-pattern") - template = assertions.Template.from_stack(stack) - - test_app = core.App(context= { - "topic_sql": "SELECT * FROM 'Opensearch_demo'", - "opensearch_domain_name": "cdk-opensearch-domain", - "opensearch_index_name": "index", - "opensearch_type_name": "_doc", - "iot_to_opensearch_rule_name": "cdk_iot_opensearchFineGrained_rule", - "master_user_role_name": "masterUserRole", - "cognito_user_pool_name" : "CDKUserPool", - "cognito_identity_pool_name": "CDK#IdentityPool", - "cognito_user_pool_domain_name": "cdk-iot-domain", - "cognito_user_username": "admin" - }) - with pytest.raises(Exception, match=r"Wrong format for the Cognito IdentityPool name. .*"): - stack = OpenSearchPatternStack(test_app, "open-search-pattern") - template = assertions.Template.from_stack(stack) - -def test_wrong_iot_rule_name(): - test_app = core.App(context= { - "topic_sql": "SELECT * FROM 'Opensearch_demo'", - "opensearch_domain_name": "cdk-opensearch-domain", - "opensearch_index_name": "index", - "opensearch_type_name": "_doc", - "iot_to_opensearch_rule_name": "cdk_@iot_opensearchFineGrained_rule", - "master_user_role_name": "masterUserRole", - "cognito_user_pool_name" : "CDKUserPool", - "cognito_identity_pool_name": "CDKIdentityPool", - "cognito_user_pool_domain_name": "cdk-iot-domain", - "cognito_user_username": "admin" - }) - with pytest.raises(Exception, match=r"The IoT topic rule name .*"): - stack = OpenSearchPatternStack(test_app, "open-search-pattern") - template = assertions.Template.from_stack(stack) - -def test_wrong_iam_role_name(): - test_app = core.App(context= { - "topic_sql": "SELECT * FROM 'Opensearch_demo'", - "opensearch_domain_name": "cdk-opensearch-domain", - "opensearch_index_name": "index", - "opensearch_type_name": "_doc", - "iot_to_opensearch_rule_name": "cdk_iot_opensearchFineGrained_rule", - "master_user_role_name": "#masterUserRole", - "cognito_user_pool_name" : "CDKUserPool", - "cognito_identity_pool_name": "CDKIdentityPool", - "cognito_user_pool_domain_name": "cdk-iot-domain", - "cognito_user_username": "admin" - }) - with pytest.raises(Exception, match=r"The IAM role name .*"): - stack = OpenSearchPatternStack(test_app, "open-search-pattern") - template = assertions.Template.from_stack(stack) - - test_app = core.App(context= { - "topic_sql": "SELECT * FROM 'Opensearch_demo'", - "opensearch_domain_name": "cdk-opensearch-domain", - "opensearch_index_name": "index", - "opensearch_type_name": "_doc", - "iot_to_opensearch_rule_name": "cdk_iot_opensearchFineGrained_rule", - "master_user_role_name": "c" * 65, - "cognito_user_pool_name" : "CDKUserPool", - "cognito_identity_pool_name": "CDKIdentityPool", - "cognito_user_pool_domain_name": "cdk-iot-domain", - "cognito_user_username": "admin" - }) - with pytest.raises(Exception, match=r"The length of the IAM role name string should not exceed 64 characters."): - stack = OpenSearchPatternStack(test_app, "open-search-pattern") - template = assertions.Template.from_stack(stack) \ No newline at end of file diff --git a/Cloud_Templates/AWS_CDK/TimestreamPattern/.gitignore b/Cloud_Templates/AWS_CDK/TimestreamPattern/.gitignore deleted file mode 100644 index 3037faa..0000000 --- a/Cloud_Templates/AWS_CDK/TimestreamPattern/.gitignore +++ /dev/null @@ -1,9 +0,0 @@ -*.swp -__pycache__ -.pytest_cache -.venv -*.egg-info - -# CDK asset staging directory -.cdk.staging -cdk.out diff --git a/Cloud_Templates/AWS_CDK/TimestreamPattern/README.md b/Cloud_Templates/AWS_CDK/TimestreamPattern/README.md deleted file mode 100644 index 07513f1..0000000 --- a/Cloud_Templates/AWS_CDK/TimestreamPattern/README.md +++ /dev/null @@ -1,102 +0,0 @@ - -# Welcome to your CDK project! -# IoT Data visulaization with Amazon Timestream - -The `cdk.json` file tells the CDK Toolkit how to execute your app. - -This project is set up like a standard Python project. The initialization -process also creates a virtualenv within this project, stored under the `.venv` -directory. To create the virtualenv it assumes that there is a `python3` -(or `python` for Windows) executable in your path with access to the `venv` -package. If for any reason the automatic creation of the virtualenv fails, -you can create the virtualenv manually. - -To manually create a virtualenv on MacOS and Linux: - -``` -$ python3 -m venv .venv -``` - -After the init process completes and the virtualenv is created, you can use the following -step to activate your virtualenv. - -``` -$ source .venv/bin/activate -``` - -If you are a Windows platform, you would activate the virtualenv like this: - -``` -% .venv\Scripts\activate.bat -``` - -Once the virtualenv is activated, you can install the required dependencies. - -``` -$ pip install -r requirements.txt -``` - -At this point you can now synthesize the CloudFormation template for this code. - -``` -$ cdk synth -``` - -To add additional dependencies, for example other CDK libraries, just add -them to your `setup.py` file and rerun the `pip install -r requirements.txt` -command. - -## Useful commands - - * `cdk ls` list all stacks in the app - * `cdk synth` emits the synthesized CloudFormation template - * `cdk deploy` deploy this stack to your default AWS account/region - * `cdk diff` compare deployed stack with current state - * `cdk docs` open CDK documentation - -## Context parameters -There are multiple context parameters that you need to set before synthesizing or delpoying this CDK stack. You can specify a context variable either as part of an AWS CDK CLI command, or in `cdk.json`. -To create a command line context variable, use the __--context (-c) option__, as shown in the following example. - -``` -$ cdk cdk synth -c bucket_name=mybucket -``` - -To specify the same context variable and value in the cdk.json file, use the following sample code. - -``` -{ - "context": { - "bucket_name": "mybucket" - } -} -``` - -In this project, these are the following parameters to be set: - -* `topic_sql` -
It is required for IoT Core rule creation to add a simplified SQL syntax to filter messages received on an MQTT topic and push the data elsewhere. -
__Format__: Enter an SQL statement using the following: ```SELECT FROM WHERE ```. For example: ```SELECT temperature FROM 'iot/topic' WHERE temperature > 50```. To learn more, see AWS IoT SQL Reference. - -* `dimensions` -
Each record contains an array of dimensions (minimum 1). Dimensions represent the metadata attributes of a time series data point. Specify the dimension(s) for your data. -
__Format__: Must be in a format of a list of strings. For example, for the input ```[device_id]``` the following key-value would be attached to the IoT Core rule: -
```{dimension's name: device_id, dimension_value: ${device_id}}```
- -* `timestream_db_name`    `` -
The name of Timestream databse to hold your data. -
__Format__: Specify a name that is unique for all Timestream databases in your AWS account in the current Region. You can not change this name once you create it. Must be between 3 and 256 characters long. Must contain letters, digits, dashes, periods or underscores. - -* `timestream_table_name`    `` -
The name of Timestream databse to hold your data. -
__Format__: Specify a table name that is unique within its database. You can not change this name once you create it. Must be between 3 and 256 characters long. Must contain letters, digits, dashes, periods or underscores. - -* `timestream_iot_rule_name`    `` -
The name of the IoT Core rule that is going to be created. -
__Format__: Should be an alphanumeric string that can also contain underscore (_) characters, but no spaces. - -* `timestream_iot_role_name`    `` -
An IAM role should be created to grant AWS IoT access to your endpoint. This parameter is for setting the name of this role. -
__Format__: Enter a unique role name that contains alphanumeric characters, hyphens, and underscores. A role name can't contain any spaces. - -Enjoy! \ No newline at end of file diff --git a/Cloud_Templates/AWS_CDK/TimestreamPattern/app.py b/Cloud_Templates/AWS_CDK/TimestreamPattern/app.py deleted file mode 100644 index b61e640..0000000 --- a/Cloud_Templates/AWS_CDK/TimestreamPattern/app.py +++ /dev/null @@ -1,27 +0,0 @@ -import os - -import aws_cdk as cdk - -from timestream_pattern.timestream_pattern_stack import TimestreamPatternStack - - -app = cdk.App() -TimestreamPatternStack(app, "TimestreamPatternStack", - # If you don't specify 'env', this stack will be environment-agnostic. - # Account/Region-dependent features and context lookups will not work, - # but a single synthesized template can be deployed anywhere. - - # Uncomment the next line to specialize this stack for the AWS Account - # and Region that are implied by the current CLI configuration. - - #env=cdk.Environment(account=os.getenv('CDK_DEFAULT_ACCOUNT'), region=os.getenv('CDK_DEFAULT_REGION')), - - # Uncomment the next line if you know exactly what Account and Region you - # want to deploy the stack to. */ - - #env=cdk.Environment(account='123456789012', region='us-east-1'), - - # For more information, see https://docs.aws.amazon.com/cdk/latest/guide/environments.html - ) - -app.synth() diff --git a/Cloud_Templates/AWS_CDK/TimestreamPattern/cdk.json b/Cloud_Templates/AWS_CDK/TimestreamPattern/cdk.json deleted file mode 100644 index 3b9e8fd..0000000 --- a/Cloud_Templates/AWS_CDK/TimestreamPattern/cdk.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "app": "python3 app.py", - "watch": { - "include": [ - "**" - ], - "exclude": [ - "README.md", - "cdk*.json", - "requirements*.txt", - "source.bat", - "**/__init__.py", - "python/__pycache__", - "tests" - ] - }, - "context": { - "@aws-cdk/aws-apigateway:usagePlanKeyOrderInsensitiveId": true, - "@aws-cdk/core:stackRelativeExports": true, - "@aws-cdk/aws-rds:lowercaseDbIdentifier": true, - "@aws-cdk/aws-lambda:recognizeVersionProps": true, - "@aws-cdk/aws-lambda:recognizeLayerVersion": true, - "@aws-cdk/aws-cloudfront:defaultSecurityPolicyTLSv1.2_2021": true, - "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true, - "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true, - "@aws-cdk/core:checkSecretUsage": true, - "@aws-cdk/aws-iam:minimizePolicies": true, - "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true, - "@aws-cdk/core:validateSnapshotRemovalPolicy": true, - "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true, - "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true, - "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true, - "@aws-cdk/core:target-partitions": [ - "aws", - "aws-cn" - ], - "topic_sql": "SELECT temperature, pressure, humidity FROM 'Timestream_demo'", - "dimensions": ["device_id"], - "timestream_db_name": "demo_db", - "timestream_table_name": "demo_table", - "timestream_iot_role_name": "demo_iot_timestream_role", - "timestream_iot_rule_name": "demo_to_timetream_rule" - } -} diff --git a/Cloud_Templates/AWS_CDK/TimestreamPattern/requirements-dev.txt b/Cloud_Templates/AWS_CDK/TimestreamPattern/requirements-dev.txt deleted file mode 100644 index 9270945..0000000 --- a/Cloud_Templates/AWS_CDK/TimestreamPattern/requirements-dev.txt +++ /dev/null @@ -1 +0,0 @@ -pytest==6.2.5 diff --git a/Cloud_Templates/AWS_CDK/TimestreamPattern/requirements.txt b/Cloud_Templates/AWS_CDK/TimestreamPattern/requirements.txt deleted file mode 100644 index 0822bbe..0000000 --- a/Cloud_Templates/AWS_CDK/TimestreamPattern/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -aws-cdk-lib==2.37.1 -constructs>=10.0.0,<11.0.0 diff --git a/Cloud_Templates/AWS_CDK/TimestreamPattern/source.bat b/Cloud_Templates/AWS_CDK/TimestreamPattern/source.bat deleted file mode 100644 index 9e1a834..0000000 --- a/Cloud_Templates/AWS_CDK/TimestreamPattern/source.bat +++ /dev/null @@ -1,13 +0,0 @@ -@echo off - -rem The sole purpose of this script is to make the command -rem -rem source .venv/bin/activate -rem -rem (which activates a Python virtualenv on Linux or Mac OS X) work on Windows. -rem On Windows, this command just runs this batch file (the argument is ignored). -rem -rem Now we don't need to document a Windows command for activating a virtualenv. - -echo Executing .venv\Scripts\activate.bat for you -.venv\Scripts\activate.bat diff --git a/Cloud_Templates/AWS_CDK/TimestreamPattern/tests/__init__.py b/Cloud_Templates/AWS_CDK/TimestreamPattern/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/Cloud_Templates/AWS_CDK/TimestreamPattern/tests/unit/__init__.py b/Cloud_Templates/AWS_CDK/TimestreamPattern/tests/unit/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/Cloud_Templates/AWS_CDK/TimestreamPattern/tests/unit/test_timestream_pattern_stack.py b/Cloud_Templates/AWS_CDK/TimestreamPattern/tests/unit/test_timestream_pattern_stack.py deleted file mode 100644 index 80fd3df..0000000 --- a/Cloud_Templates/AWS_CDK/TimestreamPattern/tests/unit/test_timestream_pattern_stack.py +++ /dev/null @@ -1,339 +0,0 @@ -import aws_cdk as core -import aws_cdk.assertions as assertions -from aws_cdk.assertions import Match -import pytest - -from timestream_pattern.timestream_pattern_stack import TimestreamPatternStack - -# Setting the context for the app -app = core.App(context={ - "topic_sql": "SELECT temperature, pressure, humidity FROM 'EL-timestream_test'", - "dimensions": ["device_id"], - "timestream_db_name": "cdk_db", - "timestream_table_name": "cdk_table", - "timestream_iot_role_name": "cdk_timestream_role", - "timestream_iot_rule_name": "cdk_to_timetream_rule" -}) - -stack = TimestreamPatternStack(app, "timestream-pattern") -template = assertions.Template.from_stack(stack) - -# Defining Capture objects for obtaining values in tests -table_ref = assertions.Capture() -db_ref = assertions.Capture() -policy_ref = assertions.Capture() -role_ref = assertions.Capture() - - -# Testing the resources' creation and properties - -def test_timestream_database_creation(): - template.has_resource("AWS::Timestream::Database", {"DeletionPolicy":"Delete", "UpdateReplacePolicy":"Delete"}) - template.resource_count_is("AWS::Timestream::Database", 1) - -def test_timestream_database_properties(): - template.has_resource_properties("AWS::Timestream::Database", { - "DatabaseName": app.node.try_get_context("timestream_db_name") - }) - -def test_timestream_table_creation(): - template.has_resource("AWS::Timestream::Table", {"DeletionPolicy":"Delete", "UpdateReplacePolicy":"Delete"}) - template.resource_count_is("AWS::Timestream::Table", 1) - -def test_timestream_table_properties(): - template.has_resource_properties("AWS::Timestream::Table", { - "DatabaseName": app.node.try_get_context("timestream_db_name"), - "TableName" : app.node.try_get_context("timestream_table_name"), - "RetentionProperties": { "MemoryStoreRetentionPeriodInHours": "24", "MagneticStoreRetentionPeriodInDays": "7"} - }) - -def test_timesream_role_creation(): - template.has_resource("AWS::IAM::Role", {"DeletionPolicy":"Delete", "UpdateReplacePolicy":"Delete"}) - -def test_timestream_role_properties(): - template.has_resource_properties("AWS::IAM::Role", { - "AssumeRolePolicyDocument": {"Statement": [{ "Action": "sts:AssumeRole", "Effect": "Allow", - "Principal": { "Service": "iot.amazonaws.com"}}], "Version": Match.any_value()} - }) - -def test_timestream_role_policy_properties(): - template.has_resource_properties("AWS::IAM::Policy", { - "PolicyDocument": { - "Statement": [ - { - "Action": "timestream:WriteRecords", - "Effect": "Allow", - "Resource": { - "Fn::GetAtt": [ - table_ref, - "Arn" - ] - } - }, - { - "Action": "timestream:DescribeEndpoints", - "Effect": "Allow", - "Resource": "*" - } - ], - "Version": Match.any_value() - }, - "PolicyName": policy_ref, - "Roles": [ - { - "Ref": role_ref - } - ] - }) - -def test_iot_topic_rule_creation(): - template.has_resource("AWS::IoT::TopicRule", {"DeletionPolicy":"Delete", "UpdateReplacePolicy":"Delete"}) - template.resource_count_is("AWS::IoT::TopicRule", 1) - -def test_iot_topic_rule_properties(): - dimesnsion_list = [] - for d in app.node.try_get_context("dimensions"): - dimesnsion_list.append({ - "Name": d, - "Value": "${" + d + "}" - }) - template.has_resource_properties("AWS::IoT::TopicRule", { - "TopicRulePayload": { - "Actions": [{ - "Timestream": { - "DatabaseName": app.node.try_get_context("timestream_db_name"), - "Dimensions": dimesnsion_list, - "RoleArn": { - "Fn::GetAtt": [ - role_ref.as_string(), - "Arn" - ] - }, - "TableName": app.node.try_get_context("timestream_table_name") - } - }], - "Sql": app.node.try_get_context("topic_sql") - } - }) - -# Testing dependencies between the resources - -def test_timestream_table_dependencies(): - template.has_resource("AWS::Timestream::Table", { - "DependsOn": [ - db_ref - ] - }) - -def test_timestream_role_dependencies(): - template.has_resource("AWS::IAM::Role", { - "DependsOn": [ - table_ref.as_string() - ] - }) - -def test_timestream_policy_dependencies(): - template.has_resource("AWS::IAM::Policy", { - "DependsOn": [ - table_ref.as_string() - ] - }) - -def test_iot_topic_rule_dependencies(): - template.has_resource("AWS::IoT::TopicRule", { - "DependsOn": [ - db_ref.as_string(), - table_ref.as_string(), - policy_ref.as_string(), - role_ref.as_string() - ] - }) - -# Testing input validation process - -def test_no_sql(): - test_app = core.App(context= { - "topic_sql": "", - "dimensions": ["device_id"], - "timestream_db_name": "cdk_db", - "timestream_table_name": "cdk_table", - "timestream_iot_role_name": "cdk_timestream_role", - "timestream_iot_rule_name": "cdk_to_timetream_rule" - }) - with pytest.raises(Exception, match=r"No sql statemtnt .*"): - stack = TimestreamPatternStack(test_app, "timestream-pattern") - template = assertions.Template.from_stack(stack) - -def test_no_dimension(): - test_app = core.App(context= { - "topic_sql": "SELECT * FROM 'EL-timestream_test'", - "timestream_db_name": "cdk_db", - "timestream_table_name": "cdk_table", - "timestream_iot_role_name": "cdk_timestream_role", - "timestream_iot_rule_name": "cdk_to_timetream_rule" - }) - with pytest.raises(Exception, match=r"No dimesnsion is provided. *"): - stack = TimestreamPatternStack(test_app, "timestream-pattern") - template = assertions.Template.from_stack(stack) - -def test_wrong_dimension(): - test_app = core.App(context= { - "topic_sql": "SELECT * FROM 'EL-timestream_test'", - "dimensions": "device_id", - "timestream_db_name": "cdk_db", - "timestream_table_name": "cdk_table", - "timestream_iot_role_name": "cdk_timestream_role", - "timestream_iot_rule_name": "cdk_to_timetream_rule" - }) - with pytest.raises(Exception, match=r"The provided input for the dimesnion list is not of type list."): - stack = TimestreamPatternStack(test_app, "timestream-pattern") - template = assertions.Template.from_stack(stack) - - test_app = core.App(context= { - "topic_sql": "SELECT * FROM 'EL-timestream_test'", - "dimensions": [2,"id"], - "timestream_db_name": "cdk_db", - "timestream_table_name": "cdk_table", - "timestream_iot_role_name": "cdk_timestream_role", - "timestream_iot_rule_name": "cdk_to_timetream_rule" - }) - with pytest.raises(Exception, match=r"At least one of the provided dimensions is not of type string."): - stack = TimestreamPatternStack(test_app, "timestream-pattern") - template = assertions.Template.from_stack(stack) - -def test_wrong_sql(): - test_app = core.App(context= { - "topic_sql": ["SELECT * FROM 'EL-timestream_test'"], - "dimensions": ["device_id"], - "timestream_db_name": "cdk_db", - "timestream_table_name": "cdk_table", - "timestream_iot_role_name": "cdk_timestream_role", - "timestream_iot_rule_name": "cdk_to_timetream_rule" - }) - with pytest.raises(Exception, match=r"The input sql statement does not have a right format. *"): - stack = TimestreamPatternStack(test_app, "timestream-pattern") - template = assertions.Template.from_stack(stack) - -def test_wrong_db_name(): - test_app = core.App(context= { - "topic_sql": "SELECT * FROM 'EL-timestream_test'", - "dimensions": ["device_id"], - "timestream_db_name": True, - "timestream_table_name": "cdk_table", - "timestream_iot_role_name": "cdk_timestream_role", - "timestream_iot_rule_name": "cdk_to_timetream_rule" - }) - with pytest.raises(Exception, match=r"The provided input for Timestream resource name is not of type string."): - stack = TimestreamPatternStack(test_app, "timestream-pattern") - template = assertions.Template.from_stack(stack) - - test_app = core.App(context= { - "topic_sql": "SELECT * FROM 'EL-timestream_test'", - "dimensions": ["device_id"], - "timestream_db_name": "db", - "timestream_table_name": "cdk_table", - "timestream_iot_role_name": "cdk_timestream_role", - "timestream_iot_rule_name": "cdk_to_timetream_rule" - }) - with pytest.raises(Exception, match=r"String length error for Timestream resource name: *"): - stack = TimestreamPatternStack(test_app, "timestream-pattern") - template = assertions.Template.from_stack(stack) - - test_app = core.App(context= { - "topic_sql": "SELECT * FROM 'EL-timestream_test'", - "dimensions": ["device_id"], - "timestream_db_name": "cdk_db!", - "timestream_table_name": "cdk_table", - "timestream_iot_role_name": "cdk_timestream_role", - "timestream_iot_rule_name": "cdk_to_timetream_rule" - }) - with pytest.raises(Exception, match=r"String format error for Timestream resource name: *"): - stack = TimestreamPatternStack(test_app, "timestream-pattern") - template = assertions.Template.from_stack(stack) - -def test_wrong_table_name(): - test_app = core.App(context= { - "topic_sql": "SELECT * FROM 'EL-timestream_test'", - "dimensions": ["device_id"], - "timestream_db_name": "cdk_db", - "timestream_table_name": ["cdk_table"], - "timestream_iot_role_name": "cdk_timestream_role", - "timestream_iot_rule_name": "cdk_to_timetream_rule" - }) - with pytest.raises(Exception, match=r"The provided input for Timestream resource name is not of type string."): - stack = TimestreamPatternStack(test_app, "timestream-pattern") - template = assertions.Template.from_stack(stack) - - test_app = core.App(context= { - "topic_sql": "SELECT * FROM 'EL-timestream_test'", - "dimensions": ["device_id"], - "timestream_db_name": "cdk_db", - "timestream_table_name": "x" * 300, - "timestream_iot_role_name": "cdk_timestream_role", - "timestream_iot_rule_name": "cdk_to_timetream_rule" - }) - with pytest.raises(Exception, match=r"String length error for Timestream resource name: *"): - stack = TimestreamPatternStack(test_app, "timestream-pattern") - template = assertions.Template.from_stack(stack) - - test_app = core.App(context= { - "topic_sql": "SELECT * FROM 'EL-timestream_test'", - "dimensions": ["device_id"], - "timestream_db_name": "cdk_db", - "timestream_table_name": "cdk_table@", - "timestream_iot_role_name": "cdk_timestream_role", - "timestream_iot_rule_name": "cdk_to_timetream_rule" - }) - with pytest.raises(Exception, match=r"String format error for Timestream resource name: *"): - stack = TimestreamPatternStack(test_app, "timestream-pattern") - template = assertions.Template.from_stack(stack) - -def test_wrong_topic_rule_name(): - test_app = core.App(context= { - "topic_sql": "SELECT * FROM 'EL-timestream_test'", - "dimensions": ["device_id"], - "timestream_db_name": "cdk_db", - "timestream_table_name": "cdk_table", - "timestream_iot_role_name": "cdk_timestream_role", - "timestream_iot_rule_name": ["cdk_to_timetream_rule"] - }) - with pytest.raises(Exception, match=r"The provided input for topic rule name is not of type string."): - stack = TimestreamPatternStack(test_app, "timestream-pattern") - template = assertions.Template.from_stack(stack) - - test_app = core.App(context= { - "topic_sql": "SELECT * FROM 'EL-timestream_test'", - "dimensions": ["device_id"], - "timestream_db_name": "cdk_db", - "timestream_table_name": "cdk_table", - "timestream_iot_role_name": "cdk_timestream_role", - "timestream_iot_rule_name": "cdk_to timetream_rule" - }) - with pytest.raises(Exception, match=r"String format error: The topic rule name *"): - stack = TimestreamPatternStack(test_app, "timestream-pattern") - template = assertions.Template.from_stack(stack) - -def test_wrong_iam_role_name(): - test_app = core.App(context= { - "topic_sql": "SELECT * FROM 'EL-timestream_test'", - "dimensions": ["device_id"], - "timestream_db_name": "cdk_db", - "timestream_table_name": "cdk_table", - "timestream_iot_role_name": "c" * 65, - "timestream_iot_rule_name": "cdk_to_timetream_rule" - }) - with pytest.raises(Exception, match=r"The length of the IAM role name string should not exceed 64 characters."): - stack = TimestreamPatternStack(test_app, "timestream-pattern") - template = assertions.Template.from_stack(stack) - - test_app = core.App(context= { - "topic_sql": "SELECT * FROM 'EL-timestream_test'", - "dimensions": ["device_id"], - "timestream_db_name": "cdk_db", - "timestream_table_name": "cdk_table", - "timestream_iot_role_name": "cdk×tream_role", - "timestream_iot_rule_name": "cdk_to_timetream_rule" - }) - with pytest.raises(Exception, match=r"String format error: The IAM role name *"): - stack = TimestreamPatternStack(test_app, "timestream-pattern") - template = assertions.Template.from_stack(stack) \ No newline at end of file diff --git a/Cloud_Templates/AWS_CDK/TimestreamPattern/timestream_pattern/__init__.py b/Cloud_Templates/AWS_CDK/TimestreamPattern/timestream_pattern/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/Cloud_Templates/AWS_CDK/TimestreamPattern/timestream_pattern/timestream_pattern_stack.py b/Cloud_Templates/AWS_CDK/TimestreamPattern/timestream_pattern/timestream_pattern_stack.py deleted file mode 100644 index 155ffa8..0000000 --- a/Cloud_Templates/AWS_CDK/TimestreamPattern/timestream_pattern/timestream_pattern_stack.py +++ /dev/null @@ -1,160 +0,0 @@ -import string -import sys -import re -from aws_cdk import ( - Stack, - aws_timestream as timestream, - aws_iot as iot, - aws_iam as iam, - aws_logs as logs -) -from constructs import Construct -import aws_cdk as cdk - -sys.path.append('../') - -from customExceptions import * - -class TimestreamPatternStack(Stack): - - # Defining class variables - dimensions_list = [] - topic_sql = "" - timestream_db_name = "" - timestream_table_name = "" - timestream_iot_role_name = "" - timestream_iot_rule_name = "" - - - def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: - super().__init__(scope, construct_id, **kwargs) - - # Getting the context parameters - - # Required parameters for users to set in the CLI command or cdk.json - self.dimensions_list = self.node.try_get_context("dimensions") - self.topic_sql = self.node.try_get_context("topic_sql") - - # Optional parameters for users to set in the CLI command or cdk.json - self.timestream_db_name = self.node.try_get_context("timestream_db_name") - self.timestream_table_name = self.node.try_get_context("timestream_table_name") - self.timestream_iot_role_name = self.node.try_get_context("timestream_iot_role_name") - self.timestream_iot_rule_name = self.node.try_get_context("timestream_iot_rule_name") - - # Perform input validation - self.performInputValidation() - - # Creating the timestream database - timestream_database = timestream.CfnDatabase(self, self.timestream_db_name, database_name=self.timestream_db_name) - timestream_database.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) - - # Creating the timestream table under the database previously made - timestream_table = timestream.CfnTable(self, self.timestream_table_name, database_name=self.timestream_db_name, - retention_properties={"MemoryStoreRetentionPeriodInHours": "24", "MagneticStoreRetentionPeriodInDays": "7"}, table_name=self.timestream_table_name) - timestream_table.node.add_dependency(timestream_database) - timestream_table.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) - - # Creating the role for the IoT-Timestream rule - iot_timestream_role = iam.Role(self, self.timestream_iot_role_name, assumed_by=iam.ServicePrincipal("iot.amazonaws.com")) - iot_timestream_role.add_to_policy(iam.PolicyStatement(effect=iam.Effect.ALLOW, resources=[timestream_table.attr_arn], actions=["timestream:WriteRecords"])) - iot_timestream_role.add_to_policy(iam.PolicyStatement(effect=iam.Effect.ALLOW, resources=["*"], actions=["timestream:DescribeEndpoints"])) - iot_timestream_role.node.add_dependency(timestream_table) - iot_timestream_role.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) - - # Creating the dimension list based on the user input - dimensions = [iot.CfnTopicRule.TimestreamDimensionProperty(name = dim, value = "${" + dim + "}") for dim in self.dimensions_list] - - # Creating a cloudwatch log group for topic rule's error action - log_group = logs.LogGroup(self, "iot_to_timestream_log_group" , log_group_name="iot_to_timestream_log_group", removal_policy=cdk.RemovalPolicy.DESTROY) - - iot_to_cloudwatch_logs_role = iam.Role(self, "iot_to_log_group_role", assumed_by=iam.ServicePrincipal("iot.amazonaws.com")) - iot_to_cloudwatch_logs_role.add_to_policy(iam.PolicyStatement( - effect=iam.Effect.ALLOW, resources=[log_group.log_group_arn], - actions=["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents", "logs:PutMetricFilter", "logs:PutRetentionPolicy"])) - iot_to_cloudwatch_logs_role.node.add_dependency(log_group) - iot_to_cloudwatch_logs_role.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) - - # Creating the IoT Topic Rule - topic_rule = iot.CfnTopicRule(self, self.timestream_iot_rule_name, topic_rule_payload=iot.CfnTopicRule.TopicRulePayloadProperty( - actions=[iot.CfnTopicRule.ActionProperty(timestream=iot.CfnTopicRule.TimestreamActionProperty( - database_name=self.timestream_db_name, - dimensions=dimensions, - role_arn=iot_timestream_role.role_arn, - table_name=self.timestream_table_name - ))], - sql=self.topic_sql, - error_action= iot.CfnTopicRule.ActionProperty( - cloudwatch_logs=iot.CfnTopicRule.CloudwatchLogsActionProperty( - log_group_name=log_group.log_group_name, - role_arn=iot_to_cloudwatch_logs_role.role_arn - ) - ))) - topic_rule.node.add_dependency(timestream_database) - topic_rule.node.add_dependency(iot_timestream_role) - topic_rule.node.add_dependency(timestream_table) - topic_rule.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) - - - def performInputValidation(self): - self.validateSql(self.topic_sql) - self.validateDimensionList(self.dimensions_list) - if not self.timestream_db_name: - self.timestream_db_name = "DemoTimestreamDB" - else: - self.validateTimestreamResourceName(self.timestream_db_name) - if not self.timestream_table_name: - self.timestream_db_name = "DemoTimestreamTable" - else: - self.validateTimestreamResourceName(self.timestream_table_name) - self.validateIoTtoTimestreamRoleName(self.timestream_iot_role_name) - self.validateIoTTpoicRuleName(self.timestream_iot_rule_name) - - def validateSql(self, sqlStatement): - if not sqlStatement: - raise NoSQL - elif type(sqlStatement) != str: - raise WrongFormattedInput("The input sql statement does not have a right format. Please refer to README.md for more information.") - return - - def validateTimestreamResourceName(self, inputStr): - if type(inputStr) != str: - raise WrongFormattedInput("The provided input for Timestream resource name is not of type string.") - elif not re.match(r'^[a-zA-Z0-9-_\.]+$', inputStr): - raise WrongFormattedInput("String format error for Timestream resource name: Must contain letters, digits, dashes, periods or underscores.") - elif len(inputStr) < 3 or len(inputStr) > 256: - raise WrongLengthForInput("String length error for Timestream resource name: Must have a minimum length of 3 and a Maximum length of 256.") - else: - return - - def validateIoTTpoicRuleName(self, inputStr): - if not inputStr: - self.timestream_iot_rule_name = "DemoIoTtoTimestreamRule" - elif type(inputStr) != str: - raise WrongFormattedInput("The provided input for topic rule name is not of type string.") - elif not re.match(r'^[a-zA-Z0-9_]+$', inputStr): - raise WrongFormattedInput("String format error: The topic rule name should be an alphanumeric string that can also contain underscore (_) characters, but no spaces.") - else: - return - - def validateIoTtoTimestreamRoleName(self, inputStr): - if not inputStr: - self.timestream_iot_role_name = "DemoIoTtoTimestreamRole" - elif type(inputStr) != str: - raise WrongFormattedInput("The provided input for the IAM role name is not of type string") - elif len(inputStr) > 64: - raise WrongLengthForInput("The length of the IAM role name string should not exceed 64 characters.") - elif not re.match(r'^[a-zA-Z0-9+=,@-_\.]+$', inputStr): - raise WrongFormattedInput("String format error: The IAM role name should be an alphanumeric string that can also contain '+=,.@-_' characters.") - else: - return - - def validateDimensionList(self, dimensinList): - if not dimensinList: - raise NoTimestreamDimension - elif type(dimensinList) != list: - raise WrongFormattedInput("The provided input for the dimesnion list is not of type list.") - else: - for d in dimensinList: - if type(d) != str: - raise WrongFormattedInput("At least one of the provided dimensions is not of type string.") - return \ No newline at end of file