From efb705ae265a442c6fcd26a549c47e1c5d6739c5 Mon Sep 17 00:00:00 2001 From: Mohanna Shahrad Date: Fri, 26 Aug 2022 23:21:38 +0000 Subject: [PATCH] Added the new kinesis branch - no conflict with timestream --- .../aws_cdk/KinesisPattern/.gitignore | 9 + .../aws_cdk/KinesisPattern/README.md | 111 +++ cloud_templates/aws_cdk/KinesisPattern/app.py | 27 + .../aws_cdk/KinesisPattern/cdk.json | 47 + .../kinesis_pattern/__init__.py | 0 .../kinesis_pattern/kinesis_pattern_stack.py | 219 +++++ .../KinesisPattern/requirements-dev.txt | 1 + .../aws_cdk/KinesisPattern/requirements.txt | 2 + .../aws_cdk/KinesisPattern/source.bat | 13 + .../aws_cdk/KinesisPattern/tests/__init__.py | 0 .../KinesisPattern/tests/unit/__init__.py | 0 .../tests/unit/test_kinesis_pattern_stack.py | 552 +++++++++++ .../demo/demo_templates/kinesis_pattern.json | 868 ++++++++++++++++++ cloud_templates/user_guides/kinesis_guide.md | 225 +++++ 14 files changed, 2074 insertions(+) create mode 100644 cloud_templates/aws_cdk/KinesisPattern/.gitignore create mode 100644 cloud_templates/aws_cdk/KinesisPattern/README.md create mode 100644 cloud_templates/aws_cdk/KinesisPattern/app.py create mode 100644 cloud_templates/aws_cdk/KinesisPattern/cdk.json create mode 100644 cloud_templates/aws_cdk/KinesisPattern/kinesis_pattern/__init__.py create mode 100644 cloud_templates/aws_cdk/KinesisPattern/kinesis_pattern/kinesis_pattern_stack.py create mode 100644 cloud_templates/aws_cdk/KinesisPattern/requirements-dev.txt create mode 100644 cloud_templates/aws_cdk/KinesisPattern/requirements.txt create mode 100644 cloud_templates/aws_cdk/KinesisPattern/source.bat create mode 100644 cloud_templates/aws_cdk/KinesisPattern/tests/__init__.py create mode 100644 cloud_templates/aws_cdk/KinesisPattern/tests/unit/__init__.py create mode 100644 cloud_templates/aws_cdk/KinesisPattern/tests/unit/test_kinesis_pattern_stack.py create mode 100644 cloud_templates/demo/demo_templates/kinesis_pattern.json create mode 100644 cloud_templates/user_guides/kinesis_guide.md diff --git a/cloud_templates/aws_cdk/KinesisPattern/.gitignore b/cloud_templates/aws_cdk/KinesisPattern/.gitignore new file mode 100644 index 0000000..3037faa --- /dev/null +++ b/cloud_templates/aws_cdk/KinesisPattern/.gitignore @@ -0,0 +1,9 @@ +*.swp +__pycache__ +.pytest_cache +.venv +*.egg-info + +# CDK asset staging directory +.cdk.staging +cdk.out diff --git a/cloud_templates/aws_cdk/KinesisPattern/README.md b/cloud_templates/aws_cdk/KinesisPattern/README.md new file mode 100644 index 0000000..0a06eb6 --- /dev/null +++ b/cloud_templates/aws_cdk/KinesisPattern/README.md @@ -0,0 +1,111 @@ + +# Welcome to your CDK project! +# IoT Data visulaization with Amazon Kinesis + +The `cdk.json` file tells the CDK Toolkit how to execute your app. + +This project is set up like a standard Python project. The initialization +process also creates a virtualenv within this project, stored under the `.venv` +directory. To create the virtualenv it assumes that there is a `python3` +(or `python` for Windows) executable in your path with access to the `venv` +package. If for any reason the automatic creation of the virtualenv fails, +you can create the virtualenv manually. + +To manually create a virtualenv on MacOS and Linux: + +``` +$ python3 -m venv .venv +``` + +After the init process completes and the virtualenv is created, you can use the following +step to activate your virtualenv. + +``` +$ source .venv/bin/activate +``` + +If you are a Windows platform, you would activate the virtualenv like this: + +``` +% .venv\Scripts\activate.bat +``` + +Once the virtualenv is activated, you can install the required dependencies. + +``` +$ pip install -r requirements.txt +``` + +At this point you can now synthesize the CloudFormation template for this code. + +``` +$ cdk synth +``` + +To add additional dependencies, for example other CDK libraries, just add +them to your `setup.py` file and rerun the `pip install -r requirements.txt` +command. + +## Useful commands + + * `cdk ls` list all stacks in the app + * `cdk synth` emits the synthesized CloudFormation template + * `cdk deploy` deploy this stack to your default AWS account/region + * `cdk diff` compare deployed stack with current state + * `cdk docs` open CDK documentation + +## Context parameters +There are multiple context parameters that you need to set before synthesizing or delpoying this CDK stack. You can specify a context variable either as part of an AWS CDK CLI command, or in `cdk.json`. +To create a command line context variable, use the __--context (-c) option__, as shown in the following example. + +``` +$ cdk cdk synth -c bucket_name=mybucket +``` + +To specify the same context variable and value in the cdk.json file, use the following code. + +``` +{ + "context": { + "bucket_name": "mybucket" + } +} +``` + +In this project, these are the following parameters to be set: + +* `topic_sql` +It is required for IoT Core rule creation to add a simplified SQL syntax to filter messages received on an MQTT topic and push the data elsewhere. +
__Format__: Enter an SQL statement using the following: ```SELECT FROM WHERE ```. For example: ```SELECT temperature FROM 'iot/topic' WHERE temperature > 50```. To learn more, see AWS IoT SQL Reference. + +* `kinesis_destination_bucket_name`    `` +To specify the destination settings for your delivery stream, a s3 bucket must be created and this parameter is for setting the name of it. +
__Format__: Bucket name must be unique and must not contain spaces or uppercase letters. [See rules for bucket naming](https://docs.aws.amazon.com/console/s3/bucket-naming) + +* `kinesis_delivery_stream_name`    `` +The name of the Kinesis delivery stream to get your data and send them to the s3 bucket. +
__Format__: Acceptable characters are uppercase and lowercase letters, numbers, underscores, hyphens, and periods. + +* `kinesis_delivery_stream_role_name`    `` +An IAM role should be created to grant Firehose access to your s3 bucket. This parameter is for setting the name of this role. +
__Format__: Enter a unique role name that contains alphanumeric characters, hyphens, and underscores. A role name can't contain any spaces. + +* `kinesis_iot_rule_name`    `` +The name of the IoT Core rule that is going to be created. +
__Format__: Should be an alphanumeric string that can also contain underscore (_) characters, but no spaces. + +* `kinesis_iot_role_name`    `` +An IAM role should be created to grant AWS IoT access to your endpoint. This parameter is for setting the name of this role. +
__Format__: Enter a unique role name that contains alphanumeric characters, hyphens, and underscores. A role name can't contain any spaces. + +* `glue_crawler_name`    `` + A Glue crawler should be created to crawl the data in your s3 bucket. This parameter is for setting the name of this cralwer. + +* `glue_db_name`    `` +A Glue database should be created to connect to the crawler and store data. This parameter is for setting the name of this database. + +* `glue_crawler_role_name`    `` +An IAM role should be created to grant Glue access to your s3 bucket. This parameter is for setting the name of this role. +
__Format__: Enter a unique role name that contains alphanumeric characters, hyphens, and underscores. A role name can't contain any spaces. + +Enjoy! diff --git a/cloud_templates/aws_cdk/KinesisPattern/app.py b/cloud_templates/aws_cdk/KinesisPattern/app.py new file mode 100644 index 0000000..707bb7e --- /dev/null +++ b/cloud_templates/aws_cdk/KinesisPattern/app.py @@ -0,0 +1,27 @@ +import os + +import aws_cdk as cdk + +from kinesis_pattern.kinesis_pattern_stack import KinesisPatternStack + + +app = cdk.App() +KinesisPatternStack(app, "KinesisPatternStack", + # If you don't specify 'env', this stack will be environment-agnostic. + # Account/Region-dependent features and context lookups will not work, + # but a single synthesized template can be deployed anywhere. + + # Uncomment the next line to specialize this stack for the AWS Account + # and Region that are implied by the current CLI configuration. + + #env=cdk.Environment(account=os.getenv('CDK_DEFAULT_ACCOUNT'), region=os.getenv('CDK_DEFAULT_REGION')), + + # Uncomment the next line if you know exactly what Account and Region you + # want to deploy the stack to. */ + + #env=cdk.Environment(account='123456789012', region='us-east-1'), + + # For more information, see https://docs.aws.amazon.com/cdk/latest/guide/environments.html + ) + +app.synth() diff --git a/cloud_templates/aws_cdk/KinesisPattern/cdk.json b/cloud_templates/aws_cdk/KinesisPattern/cdk.json new file mode 100644 index 0000000..7b53217 --- /dev/null +++ b/cloud_templates/aws_cdk/KinesisPattern/cdk.json @@ -0,0 +1,47 @@ +{ + "app": "python3 app.py", + "watch": { + "include": [ + "**" + ], + "exclude": [ + "README.md", + "cdk*.json", + "requirements*.txt", + "source.bat", + "**/__init__.py", + "python/__pycache__", + "tests" + ] + }, + "context": { + "@aws-cdk/aws-apigateway:usagePlanKeyOrderInsensitiveId": true, + "@aws-cdk/core:stackRelativeExports": true, + "@aws-cdk/aws-rds:lowercaseDbIdentifier": true, + "@aws-cdk/aws-lambda:recognizeVersionProps": true, + "@aws-cdk/aws-lambda:recognizeLayerVersion": true, + "@aws-cdk/aws-cloudfront:defaultSecurityPolicyTLSv1.2_2021": true, + "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true, + "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true, + "@aws-cdk/core:checkSecretUsage": true, + "@aws-cdk/aws-iam:minimizePolicies": true, + "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true, + "@aws-cdk/core:validateSnapshotRemovalPolicy": true, + "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true, + "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true, + "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true, + "@aws-cdk/core:target-partitions": [ + "aws", + "aws-cn" + ], + "topic_sql": "SELECT *, parse_time(\"YYYY-MM-dd'T'hh:mm:ss\", timestamp()) as Time FROM 'Kinesis_demo'", + "kinesis_destination_bucket_name": "demo-kinesis-bucket", + "kinesis_delivery_stream_role_name": "demo_kinesis_delivery_stream_role", + "kinesis_delivery_stream_name": "demo_delivery_stream", + "kinesis_iot_role_name": "demo_iot_kinesis_role", + "kinesis_iot_rule_name": "demo_to_kinesis_rule", + "glue_db_name": "demo_glue_db", + "glue_crawler_role_name": "demo_glue_crawler_role", + "glue_crawler_name": "demo_glue_crawler" + } +} diff --git a/cloud_templates/aws_cdk/KinesisPattern/kinesis_pattern/__init__.py b/cloud_templates/aws_cdk/KinesisPattern/kinesis_pattern/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cloud_templates/aws_cdk/KinesisPattern/kinesis_pattern/kinesis_pattern_stack.py b/cloud_templates/aws_cdk/KinesisPattern/kinesis_pattern/kinesis_pattern_stack.py new file mode 100644 index 0000000..ac543fa --- /dev/null +++ b/cloud_templates/aws_cdk/KinesisPattern/kinesis_pattern/kinesis_pattern_stack.py @@ -0,0 +1,219 @@ +from aws_cdk import ( + Stack, + aws_s3 as s3, + aws_kinesisfirehose as kinesisfirehose, + aws_iot as iot, + aws_iam as iam, + aws_logs as logs, + aws_glue as glue +) +from constructs import Construct +import aws_cdk as cdk +import re +import sys + +sys.path.append('../') +from common.inputValidation import * + +# Defining class variables +topic_sql = "" +kinesis_destination_bucket_name = "" +kinesis_delivery_stream_role_name = "" +kinesis_delivery_stream_name = "" +kinesis_iot_role_name = "" +kinesis_iot_rule_name = "" +glue_db_name = "" +glue_crawler_role_name = "" +glue_crawler_name = "" + +class KinesisPatternStack(Stack): + + def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: + super().__init__(scope, construct_id, **kwargs) + + # Get the context parameters + + # Required parameters for users to set in the CLI command or cdk.json + self.topic_sql = self.node.try_get_context("topic_sql") + + # Optional parameters for users to set in the CLI command or cdk.json + self.kinesis_destination_bucket_name = self.node.try_get_context("kinesis_destination_bucket_name") + self.kinesis_delivery_stream_role_name = self.node.try_get_context("kinesis_delivery_stream_role_name") + self.kinesis_delivery_stream_name = self.node.try_get_context("kinesis_delivery_stream_name") + self.kinesis_iot_role_name = self.node.try_get_context("kinesis_iot_role_name") + self.kinesis_iot_rule_name = self.node.try_get_context("kinesis_iot_rule_name") + self.glue_db_name = self.node.try_get_context("glue_db_name") + self.glue_crawler_role_name = self.node.try_get_context("glue_crawler_role_name") + self.glue_crawler_name = self.node.try_get_context("glue_crawler_name") + + # Performing input validation before starting resource creation + self.performInputValidation() + + # Create a bucket for as delivery stream's destination + bucket = s3.Bucket(self, self.kinesis_destination_bucket_name, versioned=True, removal_policy=cdk.RemovalPolicy.DESTROY, auto_delete_objects=True) + + # Creating a role for the delivery stream + firehose_role = iam.Role(self, self.kinesis_delivery_stream_role_name, assumed_by=iam.ServicePrincipal("firehose.amazonaws.com")) + firehose_role.add_to_policy(iam.PolicyStatement(effect=iam.Effect.ALLOW, resources=[bucket.bucket_arn, bucket.bucket_arn + "/*"], actions=["s3:AbortMultipartUpload", + "s3:GetBucketLocation","s3:GetObject", "s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:PutObject"])) + firehose_role.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) + + # Creating a cloud watch log group to capture any errors while sending the data from the delivery stream to the bucket + log_group = logs.LogGroup(self, "Kinesis_deliverystream_to_s3_logs" , log_group_name="Kinesis_deliverystream_to_s3_logs", removal_policy=cdk.RemovalPolicy.DESTROY) + log_stream = logs.LogStream(self, "Kinesis_deliverystream_to_s3_log_stream", log_group=log_group ,log_stream_name="Kinesis_deliverystream_to_s3_log_stream", removal_policy=cdk.RemovalPolicy.DESTROY) + + # Creating the delivery stream + delivery_stream = kinesisfirehose.CfnDeliveryStream(self, self.kinesis_delivery_stream_name, delivery_stream_name=self.kinesis_delivery_stream_name, + s3_destination_configuration=kinesisfirehose.CfnDeliveryStream.S3DestinationConfigurationProperty( + bucket_arn=bucket.bucket_arn, + role_arn= firehose_role.role_arn, + # the properties below are optional + cloud_watch_logging_options=kinesisfirehose.CfnDeliveryStream.CloudWatchLoggingOptionsProperty( + enabled=True, + log_group_name=log_group.log_group_name, + log_stream_name=log_stream.log_stream_name + ) + )) + delivery_stream.node.add_dependency(bucket) + delivery_stream.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) + + # Creating the role for the IoT Kinesis rule + iot_kinesis_role = iam.Role(self, self.kinesis_iot_role_name, assumed_by=iam.ServicePrincipal("iot.amazonaws.com")) + iot_kinesis_role.add_to_policy(iam.PolicyStatement(effect=iam.Effect.ALLOW, resources=[delivery_stream.attr_arn], actions=["firehose:PutRecord", "firehose:PutRecordBatch"])) + iot_kinesis_role.node.add_dependency(delivery_stream) + iot_kinesis_role.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) + + # Creating a cloudwatch log group for topic rule's error action + rule_log_group = logs.LogGroup(self, "iot_to_kinesis_log_group" , log_group_name="iot_to_kinesis_log_group", removal_policy=cdk.RemovalPolicy.DESTROY) + + iot_to_cloudwatch_logs_role = iam.Role(self, "iot_to_kinesis_log_group_role", assumed_by=iam.ServicePrincipal("iot.amazonaws.com")) + iot_to_cloudwatch_logs_role.add_to_policy(iam.PolicyStatement( + effect=iam.Effect.ALLOW, resources=[rule_log_group.log_group_arn], + actions=["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents", "logs:PutMetricFilter", "logs:PutRetentionPolicy"])) + iot_to_cloudwatch_logs_role.node.add_dependency(rule_log_group) + iot_to_cloudwatch_logs_role.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) + + # Creating the IoT rule with action = Kinesis Firehose + topic_rule = iot.CfnTopicRule(self, self.kinesis_iot_rule_name, topic_rule_payload=iot.CfnTopicRule.TopicRulePayloadProperty( + actions=[iot.CfnTopicRule.ActionProperty( firehose=iot.CfnTopicRule.FirehoseActionProperty( + delivery_stream_name=self.kinesis_delivery_stream_name, + role_arn=iot_kinesis_role.role_arn, + + # the properties below are optional + batch_mode=False + ) + )], sql=self.topic_sql, + aws_iot_sql_version = '2016-03-23', + error_action= iot.CfnTopicRule.ActionProperty( + cloudwatch_logs=iot.CfnTopicRule.CloudwatchLogsActionProperty( + log_group_name=rule_log_group.log_group_name, + role_arn=iot_to_cloudwatch_logs_role.role_arn + ) + ))) + topic_rule.node.add_dependency(delivery_stream) + topic_rule.node.add_dependency(iot_kinesis_role) + topic_rule.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) + + #create an Athena/Glue Database + glue_database = glue.CfnDatabase( + self, + id=self.glue_db_name, + catalog_id=cdk.Aws.ACCOUNT_ID, + database_input=glue.CfnDatabase.DatabaseInputProperty( + description=f"Glue database", + name=self.glue_db_name, + ) + ) + glue_database.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) + + # Creating a role for the glue crawler + glue_s3_role = iam.Role(self, self.glue_crawler_role_name, assumed_by=iam.ServicePrincipal("glue.amazonaws.com")) + glue_s3_role.add_to_policy(iam.PolicyStatement(effect=iam.Effect.ALLOW, resources=[bucket.bucket_arn+"*"], actions=["s3:GetObject", "s3:PutObject"])) + glue_s3_role.add_managed_policy(iam.ManagedPolicy.from_aws_managed_policy_name("service-role/AWSGlueServiceRole")) + glue_s3_role.node.add_dependency(bucket) + glue_s3_role.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) + + + # Creating an Athena/Glue crawler for the s3 bucket + s3_path = f"s3://{bucket.bucket_name}" + crawler = glue.CfnCrawler(self, self.glue_crawler_name, role=glue_s3_role.role_arn, targets=glue.CfnCrawler.TargetsProperty( + s3_targets=[glue.CfnCrawler.S3TargetProperty( + path=s3_path + )]), database_name = self.glue_db_name) + crawler.node.add_dependency(glue_s3_role) + crawler.node.add_dependency(glue_database) + crawler.node.add_dependency(bucket) + crawler.apply_removal_policy(policy=cdk.RemovalPolicy.DESTROY) + + # Performing input validation on the inpute parameters of the stack + def performInputValidation(self): + self.validateTopicSQL(self.topic_sql) + self.validateBucketName(self.kinesis_destination_bucket_name) + self.validateDeliveryStreamName(self.kinesis_delivery_stream_name) + self.validateIoTRuleName(self.kinesis_iot_rule_name) + self.validateGlueDBName(self.glue_db_name) + self.validateGlueCrawlerName(self.glue_crawler_name) + self.validateIAMRoleName(self.glue_crawler_role_name, resource="crawler") + self.validateIAMRoleName(self.kinesis_iot_role_name, resource="iot") + self.validateIAMRoleName(self.kinesis_delivery_stream_role_name, resource="delivery_stream") + + def validateTopicSQL(self, input): + if not input: + raise NoSQL + elif type(input) != str: + raise WrongFormattedInput("The input sql statement does not have a right format. Please refer to README.md for more information.") + return + + def validateBucketName(self, input): + if not input: + self.kinesis_destination_bucket_name = "demo_kinesis_bucket" + elif type(input) != str: + raise WrongFormattedInput("The provided input for s3 bucket name is not of type string") + else: + checkInputLength(self, 3, 63, input, "bucket") + checkInputPattern(self, r'(?!(^xn--|-s3alias$))^[a-z0-9][a-z0-9-]{1,61}[a-z0-9]$', input, "bucket") + + def validateDeliveryStreamName(self, input): + if not input: + self.kinesis_delivery_stream_name = "demo_delivery_stream" + else: + if type(input) != str: + raise WrongFormattedInput("The provided input for delivery stream name is not of type string") + else: + checkInputPattern(self, r'^[a-zA-Z0-9-_\.]+$', input, "delivery stream") + + def validateIoTRuleName(self, input): + if not input: + self.kinesis_iot_rule_name = "demo_to_kinesis_rule" + elif type(input) != str: + raise WrongFormattedInput("The provided input for topic rule name is not of type string") + else: + checkInputPattern(self, r'^[a-zA-Z0-9-_\.]+$', input, "IoT rule") + + def validateIAMRoleName(self, input, resource): + if not input: + if resource == 'crawler': + self.glue_crawler_role_name = "demo_glue_crawler_role" + elif resource == 'iot': + self.kinesis_iot_role_name = "demo_iot_kinesis_role" + elif resource == 'delivery_stream': + self.kinesis_delivery_stream_role_name = "demo_kinesis_delivery_stream_role" + elif type(input) != str: + raise WrongFormattedInput("The provided input for the IAM role name is not of type string") + else: + checkInputLength(self, 1, 64, input, "IAM role") + checkInputPattern(self, r'^[a-zA-Z0-9+=,@-_\.]+$', input, "IAM role") + + def validateGlueDBName(self, input): + if not input: + self.glue_db_name = "demo_glue_db" + elif type(input) != str: + raise WrongFormattedInput("The provided input for the Glue database name is not of type string") + else: + checkInputLength(self, 1, 255, input, "Gue database") + + def validateGlueCrawlerName(self, input): + if not input: + self.glue_crawler_name = "demo_glue_crawler" + else: + return \ No newline at end of file diff --git a/cloud_templates/aws_cdk/KinesisPattern/requirements-dev.txt b/cloud_templates/aws_cdk/KinesisPattern/requirements-dev.txt new file mode 100644 index 0000000..9270945 --- /dev/null +++ b/cloud_templates/aws_cdk/KinesisPattern/requirements-dev.txt @@ -0,0 +1 @@ +pytest==6.2.5 diff --git a/cloud_templates/aws_cdk/KinesisPattern/requirements.txt b/cloud_templates/aws_cdk/KinesisPattern/requirements.txt new file mode 100644 index 0000000..0822bbe --- /dev/null +++ b/cloud_templates/aws_cdk/KinesisPattern/requirements.txt @@ -0,0 +1,2 @@ +aws-cdk-lib==2.37.1 +constructs>=10.0.0,<11.0.0 diff --git a/cloud_templates/aws_cdk/KinesisPattern/source.bat b/cloud_templates/aws_cdk/KinesisPattern/source.bat new file mode 100644 index 0000000..9e1a834 --- /dev/null +++ b/cloud_templates/aws_cdk/KinesisPattern/source.bat @@ -0,0 +1,13 @@ +@echo off + +rem The sole purpose of this script is to make the command +rem +rem source .venv/bin/activate +rem +rem (which activates a Python virtualenv on Linux or Mac OS X) work on Windows. +rem On Windows, this command just runs this batch file (the argument is ignored). +rem +rem Now we don't need to document a Windows command for activating a virtualenv. + +echo Executing .venv\Scripts\activate.bat for you +.venv\Scripts\activate.bat diff --git a/cloud_templates/aws_cdk/KinesisPattern/tests/__init__.py b/cloud_templates/aws_cdk/KinesisPattern/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cloud_templates/aws_cdk/KinesisPattern/tests/unit/__init__.py b/cloud_templates/aws_cdk/KinesisPattern/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cloud_templates/aws_cdk/KinesisPattern/tests/unit/test_kinesis_pattern_stack.py b/cloud_templates/aws_cdk/KinesisPattern/tests/unit/test_kinesis_pattern_stack.py new file mode 100644 index 0000000..cc7769a --- /dev/null +++ b/cloud_templates/aws_cdk/KinesisPattern/tests/unit/test_kinesis_pattern_stack.py @@ -0,0 +1,552 @@ +import aws_cdk as core +import aws_cdk.assertions as assertions +from aws_cdk.assertions import Match + +from kinesis_pattern.kinesis_pattern_stack import KinesisPatternStack +import pytest + +# Setting the context for the app +app = core.App(context={"topic_sql": "SELECT temperature, pressure, humidity FROM 'EL-kinesis-test'", + "kinesis_destination_bucket_name": "cdk-kinesis-bucket", + "kinesis_delivery_stream_role_name": "cdk_kinesis_delivery_stream_role", + "kinesis_delivery_stream_name": "cdk_delivery_stream", + "kinesis_iot_role_name": "cdk_iot_kinesis_role", + "kinesis_iot_rule_name": "cdk_to_kinesis_rule", + "glue_db_name": "cdk_glue_db", + "glue_crawler_role_name": "cdk_glue_crawler_role", + "glue_crawler_name": "cdk_glue_crawler"}) + +stack = KinesisPatternStack(app, "kinesis-pattern") +template = assertions.Template.from_stack(stack) + +# Defining Capture objects for obtaining values in tests +bucket_ref_capture = assertions.Capture() +bucket_policy_ref = assertions.Capture() +bucket_auto_delete_object_ref = assertions.Capture() +delivery_stream_policy_name = assertions.Capture() +delivery_stream_role_ref = assertions.Capture() +iot_to_kinesis_role_ref = assertions.Capture() +iot_to_kinesis_role_policy_ref = assertions.Capture() +glue_crawler_role_ref = assertions.Capture() +glue_crawler_role_policy = assertions.Capture() +log_group_ref = assertions.Capture() +delivery_stream_logicalID = assertions.Capture() +glueDB_logicalID = assertions.Capture() + +# Testing the resources' creation and properties + +def test_bucket_creation(): + template.has_resource("AWS::S3::Bucket", {"DeletionPolicy":"Delete", "UpdateReplacePolicy":"Delete"}) + template.resource_count_is("AWS::S3::Bucket",1) + +def test_bucket_properties(): + template.has_resource_properties("AWS::S3::Bucket", { + "VersioningConfiguration": {"Status": "Enabled"} + }) + +def test_bucket_policy_creation(): + template.resource_count_is("AWS::S3::BucketPolicy",1) + +def test_bucket_policy_properties(): + + template.has_resource_properties("AWS::S3::BucketPolicy", { + "Bucket": {"Ref": bucket_ref_capture}, + }) + + template.has_resource_properties("AWS::S3::BucketPolicy", { + "PolicyDocument": { + "Statement": [{ + "Action": [ "s3:GetBucket*", "s3:List*", "s3:DeleteObject*"], + "Effect": "Allow", + "Principal": Match.any_value(), + "Resource": [ + { + "Fn::GetAtt": [ + bucket_ref_capture.as_string(), + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ { + "Fn::GetAtt": [ + bucket_ref_capture.as_string(), + "Arn" + ] + }, + "/*" + ]] + }] + }], + "Version": Match.any_value() + } + }) + + +def test_delivery_stream_role_properties(): + template.has_resource_properties("AWS::IAM::Role", { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "firehose.amazonaws.com" + } + } + ], + "Version": Match.any_value() + } + }) + +def test_delivery_stream_role_policy_properties(): + template.has_resource_properties("AWS::IAM::Policy", { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:AbortMultipartUpload", + "s3:GetBucketLocation", + "s3:GetObject", + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "s3:PutObject" + ], + "Effect": "Allow", + "Resource": [ + { "Fn::GetAtt": [ bucket_ref_capture.as_string(), "Arn"]}, + { + "Fn::Join": [ + "", + [{ "Fn::GetAtt": [ bucket_ref_capture.as_string(), "Arn" ]}, + "/*" + ]]} + ] + }], + "Version": Match.any_value() + }, + "PolicyName": delivery_stream_policy_name, + "Roles": [{ "Ref": delivery_stream_role_ref }] + }) + +def test_iot_to_kinesis_role_properties(): + template.has_resource_properties("AWS::IAM::Role", { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "iot.amazonaws.com" + } + } + ], + "Version": Match.any_value() + } + }) + +def test_iot_to_kinesis_role_policy_properties(): + template.has_resource_properties("AWS::IAM::Policy", { + "PolicyDocument": { + "Statement": [{ + "Action": [ + "firehose:PutRecord", + "firehose:PutRecordBatch" + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + delivery_stream_logicalID, + "Arn" + ] + } + }], + "Version": Match.any_value() + }, + "Roles": [{ "Ref": iot_to_kinesis_role_ref }] + }) + +def test_log_group_creation(): + template.has_resource("AWS::Logs::LogGroup", {"DeletionPolicy":"Delete", "UpdateReplacePolicy":"Delete"}) + template.resource_count_is("AWS::Logs::LogGroup",2) + +def test_log_group_properties(): + template.has_resource_properties("AWS::Logs::LogGroup", { + "LogGroupName": "Kinesis_deliverystream_to_s3_logs", + "RetentionInDays": 731 + }) + +def test_log_stream_creation(): + template.has_resource("AWS::Logs::LogStream", {"DeletionPolicy":"Delete", "UpdateReplacePolicy":"Delete"}) + template.resource_count_is("AWS::Logs::LogStream",1) + +def test_log_stream_properties(): + template.has_resource_properties("AWS::Logs::LogStream", { + "LogGroupName": { + "Ref": log_group_ref + }, + "LogStreamName": "Kinesis_deliverystream_to_s3_log_stream" + }) + + +def test_delivery_stream_creation(): + template.has_resource("AWS::KinesisFirehose::DeliveryStream", {"DeletionPolicy":"Delete", "UpdateReplacePolicy":"Delete"}) + template.resource_count_is("AWS::KinesisFirehose::DeliveryStream",1) + +def test_delivery_stream_properties(): + template.has_resource_properties("AWS::KinesisFirehose::DeliveryStream", { + "DeliveryStreamName": app.node.try_get_context("kinesis_delivery_stream_name"), + "S3DestinationConfiguration": { + "BucketARN": { + "Fn::GetAtt": [ + bucket_ref_capture.as_string(), + "Arn" + ] + }, + "CloudWatchLoggingOptions": { + "Enabled": True, + "LogGroupName": { + "Ref": log_group_ref.as_string() + }, + "LogStreamName": { + "Ref": Match.any_value() + } + }, + "RoleARN": { + "Fn::GetAtt": [ + delivery_stream_role_ref.as_string(), + "Arn" + ] + } + } + }) + +def test_iot_rule_creation(): + template.has_resource("AWS::IoT::TopicRule", {"DeletionPolicy":"Delete", "UpdateReplacePolicy":"Delete" }) + template.resource_count_is("AWS::IoT::TopicRule",1) + +def test_iot_rule_properties(): + template.has_resource_properties("AWS::IoT::TopicRule", { + "TopicRulePayload": { + "Actions": [ + { + "Firehose": { + "BatchMode": False, + "DeliveryStreamName": app.node.try_get_context("kinesis_delivery_stream_name"), + "RoleArn": { + "Fn::GetAtt": [ + iot_to_kinesis_role_ref.as_string(), + "Arn" + ] + } + } + } + ], + "Sql": app.node.try_get_context("topic_sql") + } + }) + +def test_glue_database_creation(): + template.has_resource("AWS::Glue::Database", {"DeletionPolicy":"Delete", "UpdateReplacePolicy":"Delete"}) + template.resource_count_is("AWS::Glue::Database",1) + +def test_glue_database_properties(): + template.has_resource_properties("AWS::Glue::Database", { + "DatabaseInput": { + "Description": "Glue database", + "Name": app.node.try_get_context("glue_db_name") + } + }) + +def test_glue_crawler_role_properties(): + template.has_resource_properties("AWS::IAM::Role", { + "AssumeRolePolicyDocument": { + "Statement": [{ + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "glue.amazonaws.com" + } + }], + "Version": Match.any_value() + } + }) + +def test_glue_crawler_role_policy_properties(): + template.has_resource_properties("AWS::IAM::Policy", { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetObject", + "s3:PutObject" + ], + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + bucket_ref_capture.as_string(), + "Arn" + ] + }, + "*" + ] + ] + } + } + ], + "Version": Match.any_value() + }, + "Roles": [{"Ref": glue_crawler_role_ref}] + }) + +def test_glue_crawler_creation(): + template.has_resource("AWS::Glue::Crawler", {"DeletionPolicy":"Delete", "UpdateReplacePolicy":"Delete"}) + template.resource_count_is("AWS::Glue::Crawler",1) + +def test_glue_crawler_properties(): + template.has_resource_properties("AWS::Glue::Crawler", { + "Role": { + "Fn::GetAtt": [ + glue_crawler_role_ref.as_string(), + "Arn" + ] + }, + "Targets": { + "S3Targets": [ + { + "Path": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": bucket_ref_capture.as_string() + } + ] + ] + } + } + ] + }, + "DatabaseName": app.node.try_get_context("glue_db_name") + }) + +# Testing dependencies between the resources + +def test_delivery_stream_dependencies(): + template.has_resource("AWS::KinesisFirehose::DeliveryStream", { + "DependsOn": [ + bucket_auto_delete_object_ref, + bucket_policy_ref, + bucket_ref_capture.as_string() + ] + }) + +def test_delivery_stream_dependencies(): + template.has_resource("AWS::KinesisFirehose::DeliveryStream", { + "DependsOn": [ + bucket_auto_delete_object_ref, + bucket_policy_ref, + bucket_ref_capture.as_string() + ] + }) + +def test_iot_to_kinesis_role_dependencies(): + template.has_resource("AWS::IAM::Role", { + "DependsOn": [ + delivery_stream_logicalID.as_string() + ] + }) + +def test_iot_topic_rule_dependencies(): + template.has_resource("AWS::IoT::TopicRule", { + "DependsOn": [ + delivery_stream_logicalID.as_string(), + iot_to_kinesis_role_policy_ref, + iot_to_kinesis_role_ref.as_string() + ] + }) + +def test_glue_crawler_role_dependencies(): + template.has_resource("AWS::IAM::Role", { + "DependsOn": [ + bucket_auto_delete_object_ref.as_string(), + bucket_policy_ref.as_string(), + bucket_ref_capture.as_string() + ] + }) + +def test_glue_crawler_dependencies(): + template.has_resource("AWS::Glue::Crawler", { + "DependsOn": [ + glue_crawler_role_policy, + glue_crawler_role_ref.as_string(), + glueDB_logicalID, + bucket_auto_delete_object_ref.as_string(), + bucket_policy_ref.as_string(), + bucket_ref_capture.as_string() + ] + }) + +# Testing input validation process + +def test_no_sql(): + test_app = core.App(context= { + "kinesis_destination_bucket_name": "cdk-kinesis-bucket", + "kinesis_delivery_stream_role_name": "cdk_kinesis_delivery_stream_role", + "kinesis_delivery_stream_name": "cdk_delivery_stream", + "kinesis_iot_role_name": "cdk_iot_kinesis_role", + "kinesis_iot_rule_name": "cdk_to_kinesis_rule", + "glue_db_name": "cdk_glue_db", + "glue_crawler_role_name": "cdk_glue_crawler_role", + "glue_crawler_name": "cdk_glue_crawler" + }) + with pytest.raises(Exception, match=r"No sql statemtnt .*"): + stack = KinesisPatternStack(test_app, "kinesis-pattern") + template = assertions.Template.from_stack(stack) + +def test_wrong_sql_format(): + test_app = core.App(context= { + "topic_sql": ["SELECT temperature, pressure, humidity FROM 'EL-kinesis-test'"], + "kinesis_destination_bucket_name": "cdk-kinesis-bucket", + "kinesis_delivery_stream_role_name": "cdk_kinesis_delivery_stream_role", + "kinesis_delivery_stream_name": "cdk_delivery_stream", + "kinesis_iot_role_name": "cdk_iot_kinesis_role", + "kinesis_iot_rule_name": "cdk_to_kinesis_rule", + "glue_db_name": "cdk_glue_db", + "glue_crawler_role_name": "cdk_glue_crawler_role", + "glue_crawler_name": "cdk_glue_crawler" + }) + with pytest.raises(Exception, match=r"The input sql statement does not have a right format. .*"): + stack = KinesisPatternStack(test_app, "kinesis-pattern") + template = assertions.Template.from_stack(stack) + +def test_wrong_bucket_name_format(): + test_app = core.App(context= { + "topic_sql": "SELECT temperature, pressure, humidity FROM 'EL-kinesis-test'", + "kinesis_destination_bucket_name": "cdk_kinesis_bucket", + "kinesis_delivery_stream_role_name": "cdk_kinesis_delivery_stream_role", + "kinesis_delivery_stream_name": "cdk_delivery_stream", + "kinesis_iot_role_name": "cdk_iot_kinesis_role", + "kinesis_iot_rule_name": "cdk_to_kinesis_rule", + "glue_db_name": "cdk_glue_db", + "glue_crawler_role_name": "cdk_glue_crawler_role", + "glue_crawler_name": "cdk_glue_crawler" + }) + with pytest.raises(Exception, match=r"Invalid input pattern .*"): + stack = KinesisPatternStack(test_app, "kinesis-pattern") + template = assertions.Template.from_stack(stack) + + test_app = core.App(context= { + "topic_sql": "SELECT temperature, pressure, humidity FROM 'EL-kinesis-test'", + "kinesis_destination_bucket_name": "xn--bucket", + "kinesis_delivery_stream_role_name": "cdk_kinesis_delivery_stream_role", + "kinesis_delivery_stream_name": "cdk_delivery_stream", + "kinesis_iot_role_name": "cdk_iot_kinesis_role", + "kinesis_iot_rule_name": "cdk_to_kinesis_rule", + "glue_db_name": "cdk_glue_db", + "glue_crawler_role_name": "cdk_glue_crawler_role", + "glue_crawler_name": "cdk_glue_crawler" + }) + with pytest.raises(Exception, match=r"Invalid input pattern .*"): + stack = KinesisPatternStack(test_app, "kinesis-pattern") + template = assertions.Template.from_stack(stack) + + test_app = core.App(context= { + "topic_sql": "SELECT temperature, pressure, humidity FROM 'EL-kinesis-test'", + "kinesis_destination_bucket_name": "x" * 70, + "kinesis_delivery_stream_role_name": "cdk_kinesis_delivery_stream_role", + "kinesis_delivery_stream_name": "cdk_delivery_stream", + "kinesis_iot_role_name": "cdk_iot_kinesis_role", + "kinesis_iot_rule_name": "cdk_to_kinesis_rule", + "glue_db_name": "cdk_glue_db", + "glue_crawler_role_name": "cdk_glue_crawler_role", + "glue_crawler_name": "cdk_glue_crawler" + }) + with pytest.raises(Exception, match=r"Invalid input length .*"): + stack = KinesisPatternStack(test_app, "kinesis-pattern") + template = assertions.Template.from_stack(stack) + +def test_wrong_delivery_stream_format(): + test_app = core.App(context= { + "topic_sql": "SELECT temperature, pressure, humidity FROM 'EL-kinesis-test'", + "kinesis_destination_bucket_name": "cdk-kinesis-bucket", + "kinesis_delivery_stream_role_name": "cdk_kinesis_delivery_stream_role", + "kinesis_delivery_stream_name": "cdk_delivery@stream", + "kinesis_iot_role_name": "cdk_iot_kinesis_role", + "kinesis_iot_rule_name": "cdk_to_kinesis_rule", + "glue_db_name": "cdk_glue_db", + "glue_crawler_role_name": "cdk_glue_crawler_role", + "glue_crawler_name": "cdk_glue_crawler" + }) + with pytest.raises(Exception, match=r"Invalid input pattern .*"): + stack = KinesisPatternStack(test_app, "kinesis-pattern") + template = assertions.Template.from_stack(stack) + +def test_wrong_topic_rule_name_format(): + test_app = core.App(context= { + "topic_sql": "SELECT temperature, pressure, humidity FROM 'EL-kinesis-test'", + "kinesis_destination_bucket_name": "cdk-kinesis-bucket", + "kinesis_delivery_stream_role_name": "cdk_kinesis_delivery_stream_role", + "kinesis_delivery_stream_name": "cdk_delivery_stream", + "kinesis_iot_role_name": "cdk_iot_kinesis_role", + "kinesis_iot_rule_name": "cdk_to kinesis_rule", + "glue_db_name": "cdk_glue_db", + "glue_crawler_role_name": "cdk_glue_crawler_role", + "glue_crawler_name": "cdk_glue_crawler" + }) + with pytest.raises(Exception, match=r"Invalid input pattern .*"): + stack = KinesisPatternStack(test_app, "kinesis-pattern") + template = assertions.Template.from_stack(stack) + +def test_wrong_iam_role_name_format(): + test_app = core.App(context= { + "topic_sql": "SELECT temperature, pressure, humidity FROM 'EL-kinesis-test'", + "kinesis_destination_bucket_name": "cdk-kinesis-bucket", + "kinesis_delivery_stream_role_name": "cdk_kinesis_delivery_stream_role", + "kinesis_delivery_stream_name": "cdk_delivery_stream", + "kinesis_iot_role_name": "cdk_iot_kinesis_role", + "kinesis_iot_rule_name": "cdk_to_kinesis_rule", + "glue_db_name": "cdk_glue_db", + "glue_crawler_role_name": "cdk_!glue_crawler_role", + "glue_crawler_name": "cdk_glue_crawler" + }) + with pytest.raises(Exception, match=r"Invalid input pattern .*"): + stack = KinesisPatternStack(test_app, "kinesis-pattern") + template = assertions.Template.from_stack(stack) + + test_app = core.App(context= { + "topic_sql": "SELECT temperature, pressure, humidity FROM 'EL-kinesis-test'", + "kinesis_destination_bucket_name": "cdk-kinesis-bucket", + "kinesis_delivery_stream_role_name": "c" * 70, + "kinesis_delivery_stream_name": "cdk_delivery_stream", + "kinesis_iot_role_name": "cdk_iot_kinesis_role", + "kinesis_iot_rule_name": "cdk_to_kinesis_rule", + "glue_db_name": "cdk_glue_db", + "glue_crawler_role_name": "cdk_glue_crawler_role", + "glue_crawler_name": "cdk_glue_crawler" + }) + with pytest.raises(Exception, match=r"Invalid input length .*"): + stack = KinesisPatternStack(test_app, "kinesis-pattern") + template = assertions.Template.from_stack(stack) + +def test_wrong_glue_db_name_format(): + test_app = core.App(context= { + "topic_sql": "SELECT temperature, pressure, humidity FROM 'EL-kinesis-test'", + "kinesis_destination_bucket_name": "cdk-kinesis-bucket", + "kinesis_delivery_stream_role_name": "cdk_kinesis_delivery_stream_role", + "kinesis_delivery_stream_name": "cdk_delivery_stream", + "kinesis_iot_role_name": "cdk_iot_kinesis_role", + "kinesis_iot_rule_name": "cdk_to_kinesis_rule", + "glue_db_name": "c" * 256, + "glue_crawler_role_name": "cdk_glue_crawler_role", + "glue_crawler_name": "cdk_glue_crawler" + }) + with pytest.raises(Exception, match=r"Invalid input length .*"): + stack = KinesisPatternStack(test_app, "kinesis-pattern") + template = assertions.Template.from_stack(stack) \ No newline at end of file diff --git a/cloud_templates/demo/demo_templates/kinesis_pattern.json b/cloud_templates/demo/demo_templates/kinesis_pattern.json new file mode 100644 index 0000000..7db5187 --- /dev/null +++ b/cloud_templates/demo/demo_templates/kinesis_pattern.json @@ -0,0 +1,868 @@ +{ + "Resources": { + "demokinesisbucketB1B57CC6": { + "Type": "AWS::S3::Bucket", + "Properties": { + "Tags": [ + { + "Key": "aws-cdk:auto-delete-objects", + "Value": "true" + } + ], + "VersioningConfiguration": { + "Status": "Enabled" + } + }, + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete", + "Metadata": { + "aws:cdk:path": "KinesisPatternStack/demo-kinesis-bucket/Resource" + } + }, + "demokinesisbucketPolicy225051CB": { + "Type": "AWS::S3::BucketPolicy", + "Properties": { + "Bucket": { + "Ref": "demokinesisbucketB1B57CC6" + }, + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:DeleteObject*", + "s3:GetBucket*", + "s3:List*" + ], + "Effect": "Allow", + "Principal": { + "AWS": { + "Fn::GetAtt": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092", + "Arn" + ] + } + }, + "Resource": [ + { + "Fn::GetAtt": [ + "demokinesisbucketB1B57CC6", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "demokinesisbucketB1B57CC6", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + } + }, + "Metadata": { + "aws:cdk:path": "KinesisPatternStack/demo-kinesis-bucket/Policy/Resource" + } + }, + "demokinesisbucketAutoDeleteObjectsCustomResource2C41F9C7": { + "Type": "Custom::S3AutoDeleteObjects", + "Properties": { + "ServiceToken": { + "Fn::GetAtt": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderHandler9D90184F", + "Arn" + ] + }, + "BucketName": { + "Ref": "demokinesisbucketB1B57CC6" + } + }, + "DependsOn": [ + "demokinesisbucketPolicy225051CB" + ], + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete", + "Metadata": { + "aws:cdk:path": "KinesisPatternStack/demo-kinesis-bucket/AutoDeleteObjectsCustomResource/Default" + } + }, + "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Version": "2012-10-17", + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + } + } + ] + }, + "ManagedPolicyArns": [ + { + "Fn::Sub": "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + } + ] + }, + "Metadata": { + "aws:cdk:path": "KinesisPatternStack/Custom::S3AutoDeleteObjectsCustomResourceProvider/Role" + } + }, + "CustomS3AutoDeleteObjectsCustomResourceProviderHandler9D90184F": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "S3Key": "e57c1acaa363d7d2b81736776007a7091bc73dff4aeb8135627c4511a51e7dca.zip" + }, + "Timeout": 900, + "MemorySize": 128, + "Handler": "__entrypoint__.handler", + "Role": { + "Fn::GetAtt": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092", + "Arn" + ] + }, + "Runtime": "nodejs14.x", + "Description": { + "Fn::Join": [ + "", + [ + "Lambda function for auto-deleting objects in ", + { + "Ref": "demokinesisbucketB1B57CC6" + }, + " S3 bucket." + ] + ] + } + }, + "DependsOn": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092" + ], + "Metadata": { + "aws:cdk:path": "KinesisPatternStack/Custom::S3AutoDeleteObjectsCustomResourceProvider/Handler", + "aws:asset:path": "asset.e57c1acaa363d7d2b81736776007a7091bc73dff4aeb8135627c4511a51e7dca", + "aws:asset:property": "Code" + } + }, + "demokinesisdeliverystreamrole54A20861": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "firehose.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + }, + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete", + "Metadata": { + "aws:cdk:path": "KinesisPatternStack/demo_kinesis_delivery_stream_role/Resource" + } + }, + "demokinesisdeliverystreamroleDefaultPolicyE78063D3": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:AbortMultipartUpload", + "s3:GetBucketLocation", + "s3:GetObject", + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "s3:PutObject" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "demokinesisbucketB1B57CC6", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "demokinesisbucketB1B57CC6", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "demokinesisdeliverystreamroleDefaultPolicyE78063D3", + "Roles": [ + { + "Ref": "demokinesisdeliverystreamrole54A20861" + } + ] + }, + "Metadata": { + "aws:cdk:path": "KinesisPatternStack/demo_kinesis_delivery_stream_role/DefaultPolicy/Resource" + } + }, + "Kinesisdeliverystreamtos3logsE7DF0DD1": { + "Type": "AWS::Logs::LogGroup", + "Properties": { + "LogGroupName": "Kinesis_deliverystream_to_s3_logs", + "RetentionInDays": 731 + }, + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete", + "Metadata": { + "aws:cdk:path": "KinesisPatternStack/Kinesis_deliverystream_to_s3_logs/Resource" + } + }, + "Kinesisdeliverystreamtos3logstream77FC06C1": { + "Type": "AWS::Logs::LogStream", + "Properties": { + "LogGroupName": { + "Ref": "Kinesisdeliverystreamtos3logsE7DF0DD1" + }, + "LogStreamName": "Kinesis_deliverystream_to_s3_log_stream" + }, + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete", + "Metadata": { + "aws:cdk:path": "KinesisPatternStack/Kinesis_deliverystream_to_s3_log_stream/Resource" + } + }, + "demodeliverystream": { + "Type": "AWS::KinesisFirehose::DeliveryStream", + "Properties": { + "DeliveryStreamName": "demo_delivery_stream", + "S3DestinationConfiguration": { + "BucketARN": { + "Fn::GetAtt": [ + "demokinesisbucketB1B57CC6", + "Arn" + ] + }, + "CloudWatchLoggingOptions": { + "Enabled": true, + "LogGroupName": { + "Ref": "Kinesisdeliverystreamtos3logsE7DF0DD1" + }, + "LogStreamName": { + "Ref": "Kinesisdeliverystreamtos3logstream77FC06C1" + } + }, + "RoleARN": { + "Fn::GetAtt": [ + "demokinesisdeliverystreamrole54A20861", + "Arn" + ] + } + } + }, + "DependsOn": [ + "demokinesisbucketAutoDeleteObjectsCustomResource2C41F9C7", + "demokinesisbucketPolicy225051CB", + "demokinesisbucketB1B57CC6" + ], + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete", + "Metadata": { + "aws:cdk:path": "KinesisPatternStack/demo_delivery_stream" + } + }, + "demoiotkinesisrole269B3951": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "iot.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + }, + "DependsOn": [ + "demodeliverystream" + ], + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete", + "Metadata": { + "aws:cdk:path": "KinesisPatternStack/demo_iot_kinesis_role/Resource" + } + }, + "demoiotkinesisroleDefaultPolicy7B8299D4": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "firehose:PutRecord", + "firehose:PutRecordBatch" + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "demodeliverystream", + "Arn" + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "demoiotkinesisroleDefaultPolicy7B8299D4", + "Roles": [ + { + "Ref": "demoiotkinesisrole269B3951" + } + ] + }, + "DependsOn": [ + "demodeliverystream" + ], + "Metadata": { + "aws:cdk:path": "KinesisPatternStack/demo_iot_kinesis_role/DefaultPolicy/Resource" + } + }, + "iottokinesisloggroup622FA173": { + "Type": "AWS::Logs::LogGroup", + "Properties": { + "LogGroupName": "iot_to_kinesis_log_group", + "RetentionInDays": 731 + }, + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete", + "Metadata": { + "aws:cdk:path": "KinesisPatternStack/iot_to_kinesis_log_group/Resource" + } + }, + "iottokinesisloggrouprole7ACA939D": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "iot.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + }, + "DependsOn": [ + "iottokinesisloggroup622FA173" + ], + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete", + "Metadata": { + "aws:cdk:path": "KinesisPatternStack/iot_to_kinesis_log_group_role/Resource" + } + }, + "iottokinesisloggrouproleDefaultPolicyF2AF3A06": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents", + "logs:PutMetricFilter", + "logs:PutRetentionPolicy" + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "iottokinesisloggroup622FA173", + "Arn" + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "iottokinesisloggrouproleDefaultPolicyF2AF3A06", + "Roles": [ + { + "Ref": "iottokinesisloggrouprole7ACA939D" + } + ] + }, + "DependsOn": [ + "iottokinesisloggroup622FA173" + ], + "Metadata": { + "aws:cdk:path": "KinesisPatternStack/iot_to_kinesis_log_group_role/DefaultPolicy/Resource" + } + }, + "demotokinesisrule": { + "Type": "AWS::IoT::TopicRule", + "Properties": { + "TopicRulePayload": { + "Actions": [ + { + "Firehose": { + "BatchMode": false, + "DeliveryStreamName": "demo_delivery_stream", + "RoleArn": { + "Fn::GetAtt": [ + "demoiotkinesisrole269B3951", + "Arn" + ] + } + } + } + ], + "AwsIotSqlVersion": "2016-03-23", + "ErrorAction": { + "CloudwatchLogs": { + "LogGroupName": { + "Ref": "iottokinesisloggroup622FA173" + }, + "RoleArn": { + "Fn::GetAtt": [ + "iottokinesisloggrouprole7ACA939D", + "Arn" + ] + } + } + }, + "Sql": "SELECT *, parse_time(\"YYYY-MM-dd'T'hh:mm:ss\", timestamp()) as Time FROM 'Kinesis_demo'" + } + }, + "DependsOn": [ + "demodeliverystream", + "demoiotkinesisroleDefaultPolicy7B8299D4", + "demoiotkinesisrole269B3951" + ], + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete", + "Metadata": { + "aws:cdk:path": "KinesisPatternStack/demo_to_kinesis_rule" + } + }, + "demogluedb": { + "Type": "AWS::Glue::Database", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseInput": { + "Description": "Glue database", + "Name": "demo_glue_db" + } + }, + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete", + "Metadata": { + "aws:cdk:path": "KinesisPatternStack/demo_glue_db" + } + }, + "demogluecrawlerrole6DC0CF32": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "glue.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSGlueServiceRole" + ] + ] + } + ] + }, + "DependsOn": [ + "demokinesisbucketAutoDeleteObjectsCustomResource2C41F9C7", + "demokinesisbucketPolicy225051CB", + "demokinesisbucketB1B57CC6" + ], + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete", + "Metadata": { + "aws:cdk:path": "KinesisPatternStack/demo_glue_crawler_role/Resource" + } + }, + "demogluecrawlerroleDefaultPolicyB2BDDDA6": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetObject", + "s3:PutObject" + ], + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "demokinesisbucketB1B57CC6", + "Arn" + ] + }, + "*" + ] + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "demogluecrawlerroleDefaultPolicyB2BDDDA6", + "Roles": [ + { + "Ref": "demogluecrawlerrole6DC0CF32" + } + ] + }, + "DependsOn": [ + "demokinesisbucketAutoDeleteObjectsCustomResource2C41F9C7", + "demokinesisbucketPolicy225051CB", + "demokinesisbucketB1B57CC6" + ], + "Metadata": { + "aws:cdk:path": "KinesisPatternStack/demo_glue_crawler_role/DefaultPolicy/Resource" + } + }, + "demogluecrawler": { + "Type": "AWS::Glue::Crawler", + "Properties": { + "Role": { + "Fn::GetAtt": [ + "demogluecrawlerrole6DC0CF32", + "Arn" + ] + }, + "Targets": { + "S3Targets": [ + { + "Path": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "demokinesisbucketB1B57CC6" + } + ] + ] + } + } + ] + }, + "DatabaseName": "demo_glue_db" + }, + "DependsOn": [ + "demogluecrawlerroleDefaultPolicyB2BDDDA6", + "demogluecrawlerrole6DC0CF32", + "demogluedb", + "demokinesisbucketAutoDeleteObjectsCustomResource2C41F9C7", + "demokinesisbucketPolicy225051CB", + "demokinesisbucketB1B57CC6" + ], + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete", + "Metadata": { + "aws:cdk:path": "KinesisPatternStack/demo_glue_crawler" + } + }, + "CDKMetadata": { + "Type": "AWS::CDK::Metadata", + "Properties": { + "Analytics": "v2:deflate64:H4sIAAAAAAAA/1VQu27DMAz8luwym9ZD5tYFumQwnO6BIjMOa1kMRCmBIfjf61cTdLrj3YE88A3yHLYbfZfM1G1m6QTpELRp1Sgdk+SQPqJpMaji7Fa2QMmWTP+Ul3lQpDtIFVucrBmfyb+M5UYg7bn58hyvk/PgIzkEj7pb1WUYVEsOheRMHi8sCKP7iZZu6Ps1Txwm9ZuvZKo43m1sREhTUAd90jI3Kry+W/TDoIoogbsKhaM3S9sH/2eVnm9Uo1fvIhjG7zTkmnkXu5oCsRtU2YcLu5ccdvC63fwIUeajC9QhVAv+AnczsWFqAQAA" + }, + "Metadata": { + "aws:cdk:path": "KinesisPatternStack/CDKMetadata/Default" + }, + "Condition": "CDKMetadataAvailable" + } + }, + "Conditions": { + "CDKMetadataAvailable": { + "Fn::Or": [ + { + "Fn::Or": [ + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "af-south-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ap-east-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ap-northeast-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ap-northeast-2" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ap-south-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ap-southeast-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ap-southeast-2" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ca-central-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "cn-north-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "cn-northwest-1" + ] + } + ] + }, + { + "Fn::Or": [ + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "eu-central-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "eu-north-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "eu-south-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "eu-west-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "eu-west-2" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "eu-west-3" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "me-south-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "sa-east-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "us-east-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "us-east-2" + ] + } + ] + }, + { + "Fn::Or": [ + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "us-west-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "us-west-2" + ] + } + ] + } + ] + } + }, + "Parameters": { + "BootstrapVersion": { + "Type": "AWS::SSM::Parameter::Value", + "Default": "/cdk-bootstrap/hnb659fds/version", + "Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip]" + } + }, + "Rules": { + "CheckBootstrapVersion": { + "Assertions": [ + { + "Assert": { + "Fn::Not": [ + { + "Fn::Contains": [ + [ + "1", + "2", + "3", + "4", + "5" + ], + { + "Ref": "BootstrapVersion" + } + ] + } + ] + }, + "AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI." + } + ] + } + } +} diff --git a/cloud_templates/user_guides/kinesis_guide.md b/cloud_templates/user_guides/kinesis_guide.md new file mode 100644 index 0000000..02a5172 --- /dev/null +++ b/cloud_templates/user_guides/kinesis_guide.md @@ -0,0 +1,225 @@ +# Getting started with the Kinesis Data Firehose template guide + +## Setting up and prerequisites + +### AWS Account + +If you don't already have an AWS account follow [Setup Your Environment](https://aws.amazon.com/getting-started/guides/setup-environment/) for a quick overview to get started. + +### AWS CloudFormation + +Before you start using AWS CloudFormation, you might need to know what IAM permissions you need, how to start logging AWS CloudFormation API calls, or what endpoints to use. Refer to this [guide](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/settingup.html) to get started using AWS CloudFormation. + +### AWS CDK + +**Note**: If you are just going to use the sample demo template you can skip this section. + +The AWS Cloud Development Kit (CDK) is an open-source software development framework that lets you define your cloud infrastructure as code in one of its supported programming languages. It is intended for moderately to highly experienced AWS users. Refer to this [guide](https://aws.amazon.com/getting-started/guides/setup-cdk/?pg=gs&sec=gtkaws) to get started with AWS CDK. + +## Template deployment and CloudFormation stack creation + +___ + +A template is a JSON or YAML text file that contains the configuration information about the AWS resources you want to create in the [stack](https://docs.aws.amazon.com/cdk/v2/guide/stacks.html). To learn more about how to work with CloudFormation templates refer to the [Working with templates](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/template-guide.html) guide. + +You can either use the provided demo template and deploy it directly in the console or customize the template’s resources before deployment using AWS CDK. Based on your decision follow the respective section below. + +### Sample demo template + +By using the sample JSON template that is provided, you do not need to take any further actions except creating the stack by uploading the template file. For simplicity’s sake, a simple code is provided that you can run on your device. It is an example of multiple devices sending their weather measurements to the cloud through ExpressLink. You can find the code and guide to get it working under the `demo/demo_weather_station_code` directory. + +Follow the steps below to create the CloudFormation stack using the sample template file. + +1. Sign in to the AWS Management Console and open [AWS CloudFormation console.](https://console.aws.amazon.com/cloudformation) +2. If this is a new CloudFormation account, select **Create New Stack**. Otherwise, choose **Create Stack** and then select **with new resources**. +3. In the **Template** section, select **Upload a template file** and upload the JSON template file. Choose **Next**. +4. In the **Specify Details** section, enter a stack name in the **Name** field. +5. If you want you can add tags to your stack. Otherwise, select **Next**. +6. Review the stack’s settings and then select **Create.** +7. At this point, you will find the status of your stack to be `CREATE_IN_PROGRESS`. Your stack might take several minutes to get created. See the next sections to learn about monitoring your stack creation. + +### Custom template + +If you are interested in using the CloudFormation templates more than just for demo purposes, you need to customize the stack’s resources based on your specific use case. Follow the steps below to do so: + +1. Make sure that you already [set up your AWS CDK](https://aws.amazon.com/getting-started/guides/setup-cdk/?pg=gs&sec=gtkaws) environment. +2. Starting in your current directory, change your directory and go to `aws_cdk/KinesisPattern` directory. +3. Just to verify everything is working correctly, list the stacks in your app by running `cdk ls`. If you don't see `KinesisPatternStack`, make sure you are currently in the `KinesisPattern` directory. +4. The structure of the files inside `KinesisPattern` is as below: + + + +* `kinesis_pattern_stack.py` is the main code of the stack. It is here where the required resources are created. +* `tests/unit/test_kinesis_pattern_stack.py` is where the unit tests of the stack are written. The unit tests check + * Right creation of the resources in addition to their properties + * Dependencies between the resources + * Right error handlings in case of input violations +* `cdk.json` tells the CDK Toolkit how to execute your app. Context values are key-value pairs that can be associated with an app, stack, or construct. You can add the context key-values to this file or in the command line before synthesizing the template. +* `README.md` is where you can find detailed instructions on how to get started with the code including how to synthesize the template, a set of useful commands, the stack’s context parameters, and details about the code. +* `cdk.out` is where the synthesized template (in a JSON format) will be located in. + +1. Run `source .venv/bin/activate` to activate the app's Python virtual environment. +2. Run `python -m pip install -r requirements.txt` and `python -m pip install -r requirements.txt` to install the dependencies. +3. Go through the `README.md` file to learn about the context parameters that need to be set by you prior to deployment. +4. Set the context parameter values either by changing `cdk.json` file or by using the command line. + 1. To create a command line context variable, use the **`—-context (-c) option`**, as shown in the following example: `$ cdk cdk synth -c bucket_name=mybucket` + 2. To specify the same context variable and value in the `cdk.json` file, use the following code.` + {"context": { "bucket_name": "mybucket"}` +5. Run `cdk synth` to emit the synthesized CloudFormation template. +6. Run `python -m pytest` to run the unit tests. It is the best practice to run the tests before deploying your template to the cloud. +7. Run `cdk deploy` to deploy the stack to your default AWS account/region. +8. Use the instructions in the ***Stack management*** section below to manage your stack creation. + +## Stack management + +___ + +### Viewing CloudFormation stack data and resources + +After deployment, you may need to monitor your created stack and its resources. To do this, your starting point should be AWS CloudFormation. + +1. Sign in to the AWS Management Console and open [AWS CloudFormation console](https://console.aws.amazon.com/cloudformation). +2. Select the **Stacks** tab to view all the available stacks in your account. +3. Find the stack that you just created and click on it. +4. To verify that the stack’s creation is done successfully, check if its status is `CREATE_COMPLETE`. To learn more about what each status means refer to [Stack status codes](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/cfn-console-view-stack-data-resources.html#cfn-console-view-stack-data-resources-status-codes). +5. You can view the stack information such as its ID, status, policy, rollback configuration, etc under the **Stack info** tab. +6. If you click on the **Events** tab, each major step in the creation of the stack sorted by the time of each event, with the latest events on top is displayed. +7. You can also find the resources that are part of the stack under the **Resources** tab. + +There is more information about viewing stack information [here](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/cfn-console-view-stack-data-resources.html#cfn-console-view-stack-data-resources-view-info). + +### Monitoring the generated resources + +If you deploy and create the stack successfully, the following resources must get created under your stack. You can verify their creation by checking the **Resources** tab in your stack as mentioned above. + +|Resourse |Type | +|--- |--- | +|CDKMetadata |[AWS::CDK::Metadata](https://docs.aws.amazon.com/cdk/api/v1/docs/constructs.ConstructMetadata.html) | +|Kinesis Delivery Stream |[AWS::KinesisFirehose::DeliveryStream](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-kinesisfirehose-deliverystream.html) | +|Delivery stream's destination bucket (and its bucketPolicy and deleteObjects) |[AWS::S3::Bucket](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-s3-bucket.html) [AWS::S3::BucketPolicy](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-s3-policy.html) | +|IAM role and policy to grant Firehose access to S3 |[AWS::IAM::Role](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-role.html) [AWS::IAM::Policy](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-policy.html) | +|CloudWatch log group and log stream to capture error logs (delivery stream to bucket action) |[AWS::Logs::LogGroup](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-logs-loggroup.html) | +|IoT Rule |[AWS::IoT::TopicRule](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iot-topicrule.html) | +|CloudWatch log group to capture error logs (IoT Core to Kinesis action) |[AWS::Logs::LogGroup](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-logs-loggroup.html) | +|IAM role and policy that grants IoT access to Kinesis firehose |[AWS::IAM::Role](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-role.html) [AWS::IAM::Policy](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-policy.html) | +|IAM role and policy that grants IoT access to the CloudWatch log groups |[AWS::IAM::Role](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-role.html) [AWS::IAM::Policy](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-policy.html) | +|Glue database |[AWS::Glue::Database](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-glue-database.html) | +|Gue crawler |[AWS::Glue::Crawler](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-glue-crawler.html) | +|IAM role and policy that grants Glue access to S3 |[AWS::IAM::Role](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-role.html)[AWS::IAM::Policy](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-policy.html) | + +### Handling stack failures + +If CloudFormation fails to create, update, or delete your stack, you will be able to go through the logs or error messages to learn more about the issue. There are some general methods for troubleshooting a CloudFormation issue. For example, you can follow the steps below to find the issue manually in the console. + +* Check the status of your stack in the [CloudFormation console](https://console.aws.amazon.com/cloudformation/). +* From the **Events** tab, you can see a set of events while the last operation was being done on your stack. +* Find the failure event from the set of events and then check the status reason of that event. The status reason usually gives a good understanding of the issue that caused the failure. + + +In case of failures in stack creations or updates, CloudFormation automatically performs a rollback. However, you can also [add rollback triggers during stack creation or updating](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-rollback-triggers.html#using-cfn-rollback-triggers-create) to further monitor the state of your application. By setting up the rollback triggers if the application breaches the threshold of the alarms you've specified, it will roll back to that operation. + +Finally, this [troubleshooting guide](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/troubleshooting.html#basic-ts-guide) is a helpful resource to refer if there is an issue in your stack. + +### Estimating the cost of the stack + +There is no additional charge for AWS CloudFormation. You pay for AWS resources created using CloudFormation as if you created them by hand. Refer to this [guide](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-paying.html) to learn more about the stack cost estimation functionality. + +## Ingesting and visualizing your IoT data with the constructed resources + +___ + +### Sending data to the cloud from your device + +Now that your stack and all the required resources are created and available, you can start by connecting your device to the cloud and sending your data to the cloud. + +* If you are new to AWS IoT Core, this [guide](https://docs.aws.amazon.com/iot/latest/developerguide/connect-to-iot.html) is a great starting point to connect your device to the cloud. +* After connecting your device to IoT Core, you can use the [MQTT test client](https://docs.aws.amazon.com/iot/latest/developerguide/view-mqtt-messages.html) to monitor the MQTT messages being passed in your AWS account. +* Move to the **Rules** tab under the **Message Routing** section in the [AWS IoT console](https://console.aws.amazon.com/iot/home). There you can verify the creation of the newly created topic rule and its [firehose rule action](https://docs.aws.amazon.com/iot/latest/developerguide/kinesis-firehose-rule-action.html) which sends your data from MQTT messages to the Amazon Kinesis Data Firehose stream. + +### Access and query the data sent to the Kinesis delivery stream + +* Open [Amazon Kinesis console](https://console.aws.amazon.com/kinesis). +* From the left navigation pane, choose **Delivery streams.** +* Find the delivery stream that was created by your stack and select it. +* Select the **Configuration** tab. +* In the **Destination settings** section, you would be able to find the detention s3 bucket url. Click on it. +* You are now able to see your data in the form of nested folders in the bucket (partitioned data). You can download the files or continue with the rest of the steps to query them using AWS glue and Amazon Athena. (Refer to [Athena integration with AWS Glue](https://docs.aws.amazon.com/athena/latest/ug/glue-athena.html) for more detailed instructions) + * If you cannot see any data in your bucket, follow these steps: + * Simply wait for a couple of minutes. this is because Kinesis Data Firehose buffers incoming data before delivering it to the S3 destination. Therefore, it might be the buffer size or interval that is not full yet to write the data to your bucket. If after a couple of minutes, you still cannot see any data in your bucket then move to the next step. + * Go back to the [Amazon Kinesis console](https://console.aws.amazon.com/kinesis) and find your delivery stream. Under the **Test with demo data** section, click on **Start sending demo data**. This enables you to test the configuration of your delivery stream regardless of your own data availability. Check your bucket again to see if any data is being received there. If there is not, there is an issue with sending data from the delivery stream to S3. Click on the **Destination error logs** tab and open the CloudWatch log group. Select the log stream there and go through the event logs to find the issue. [Troubleshooting Kinesis Data Firehose](https://docs.aws.amazon.com/firehose/latest/dev/troubleshooting.html) is a helpful resource to help you find out the cause of the problem. + * If the test data from the previous part was successfully landed in the destination bucket, it means that the issue is with your device data being sent to the Kinesis delivery stream. To fix it first make sure that your device is connected to the cloud and is sending data by using the [MQTT test client](https://docs.aws.amazon.com/iot/latest/developerguide/view-mqtt-messages.html). More details about this are provided in the previous section. + * If your data is getting landed in IoT Core but not in the delivery stream, there might be an error happening while the IoT rule attempts to send data from IoT Core to Kinesis Data Firehose. To find out about the issue, you can use the CloudWatch log group that is created by the template earlier. To do so, open the [Cloudwatch console](https://console.aws.amazon.com/cloudwatch). From the navigation bar, select **Log > Log Groups**. Find the log group that was created for logging the IoT rules’ errors and select it. View the logs to find out the issue. +* During the CloudFormation stack deployment, a Glue crawler was created that retrieves the bucket’s data automatically and outputs the results to the Athena database and table that was created by the stack. To access this Glue crawler, open [AWS Glue console](https://console.aws.amazon.com/glue). From the navigation pane, select **Crawlers.** Find the crawler that was created earlier and select it. +* Select **Run Crawler**. +* You can see that the status of the crawler changes to **Starting**. Wait for the crawler to finish its run (i.e. its state goes back to **Ready**) +* If the run is successful, under the **Tables added** column you should see **1** which means the crawler created a glue table. (Note that if this is not your first time running the crawler, the **Tables updated** column would change) + * If there is no table added or updated, click on the **Logs** of the crawler. It navigates you to the CloudWatch log group that captures the error logs of the crawler. From there you can find the error message. [Troubleshooting AWS Glue](https://docs.aws.amazon.com/glue/latest/dg/troubleshooting-glue.html)can be a helpful guide to fix the issue. +* Open [Athena console](https://console.aws.amazon.com/athena/home). +* In the query editor, choose the glue database that was created earlier. (The data source should be **AWSDataCatalog** by default) +* Under the database, you can find the newly added glue table. +* Query the table using the editor. Now you can see your device’s data in a table format. + +### Optional steps + +* You can transform source records with AWS Lambda or convert record format. + * Open [Amazon Kinesis console](https://console.aws.amazon.com/kinesis) and from the left navigation pane, choose **Delivery streams. f**ind the delivery stream that was created by your stack and select it. + * Select the **Configuration** tab. + * Edit the **Transform and convert records** section with the settings you want to use in your application. Refer to [Record Transformation and Record Format Conversion](https://docs.aws.amazon.com/firehose/latest/dev/create-transform.html) for more information. +* Kinesis Data Firehose buffers incoming records before delivering them to your S3 bucket. Record delivery is triggered once the value of either of the specified buffering hints is reached. You can change the Kinesis buffer size or interval based on your specific use case. Additionally, you can edit the record compression and encryption settings. + * Open [Amazon Kinesis console](https://console.aws.amazon.com/kinesis) and from the left navigation pane, choose **Delivery streams**. Find the delivery stream that was created by your stack and select it. + * Select the **Configuration** tab. + * Edit **Buffer hints** and **Compression and encryption** under the **Destination settings** section. Refer to [Destination Settings](https://docs.aws.amazon.com/firehose/latest/dev/create-destination.html) for more information. +* You can change the run frequency of the Glue crawler based on your use case. By default, it is on **Run on demand** mode therefore, every time you want to update your glue database/table with new data you should manually run the crawler. You can schedule the crawler’s runs by: + * Open [AWS Glue console](https://console.aws.amazon.com/glue). From the navigation pane, select **Crawlers.** Find the demo crawler and select it. + * Select **Edit**. + * Under the **Schedule** section, select your desired frequency from the drop-down and save. + +### Integrating with dashboards to visualize data + +In the previous section, you were able to see your device’s data in a glue table format using a crawler and Athena editor. You can take a further step to visualize your data and create dashboards. Here are two possible integrations with reporting dashboards : + +#### Amazon QuickSight + +Amazon Athena provides direct integration with [Amazon QuickSight](https://aws.amazon.com/quicksight/). Amazon QuickSight is a fast business analytics service you can use to build visualizations, perform ad-hoc analysis, and quickly get business insights from your data. Amazon QuickSight is available in [these regions](https://docs.aws.amazon.com/general/latest/gr/quicksight.html). + +To connect Athena to QuickSight you need to follow these steps: + +1. Navigate to the AWS QuickSight console. +2. If you have never used AWS QuickSight before, you will be asked to sign up. In this case, choose the **Standard** tier and your region as your setup. +3. During the signup phase, give QuickSight access to your Amazon S3 buckets and Amazon Athena. +4. If you already have an account, give Amazon QuickSight access to your Athena by choosing **Admin >** **Manage QuickSight > Security & permissions.** Under QuickSight access to AWS services, choose **Add or remove**, then select the check box next to Amazon Athena and the Amazon S3 (select the buckets that are required) and choose **Update**. +5. From the admin Amazon QuickSight console page choose **New Analysis** and **New data set.** +6. Choose Athena as the source and enter a name for your data source. Let the Athena workgroup be ***[ primary ]* **and proceed. +7. Choose your Glue database and table to import. +8. After your data source is created, you can start making visualizations in Amazon QuickSight. + +#### Amazon Managed Grafana + +With Amazon Managed Grafana, you can add Athena as a data source by using the AWS data source configuration option in the Grafana workspace console. To get started refer to [Setting up](https://docs.aws.amazon.com/grafana/latest/userguide/Amazon-Managed-Grafana-setting-up.html) to set up your amazon Managed Grafana and then follow this [guide](https://docs.aws.amazon.com/grafana/latest/userguide/AWS-Athena.html) to connect Athena to your Amazon Managed Grafana. Note that there are prerequisites for Athena to be accessible to the Amazon Managed Grafana, make sure to follow these steps. + +## Cleaning up the stack + +___ + +To clean up all the resources used in this demo, all you need to do is to delete the initial CloudFormation stack. To delete a stack and its resources, follow these steps: + +1. Open [AWS CloudFormation console](https://console.aws.amazon.com/cloudformation/). +2. On the Stacks page in the CloudFormation console, select the stack that you want to delete. Note that the stack must be currently running. +3. In the stack details pane, choose **Delete**. +4. Confirm deleting stack when prompted. + +After the stack is deleted, the stack’s status will be `DELETE_COMPLETE`. Stacks in the `DELETE_COMPLETE` state aren't displayed in the CloudFormation console by default. However, you can follow the instructions in [Viewing deleted stacks on the AWS CloudFormation console](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/cfn-console-view-deleted-stacks.html) to be able to view them. + +Finally, if the stack deletion failed, the stack will be in the `DELETE_FAILED` state. For solutions, see [Delete stack fails](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/troubleshooting.html#troubleshooting-errors-delete-stack-fails) troubleshooting topic. In this case, make sure to refer to the **Monitoring the generated resources** section of this document to verify that all the resources got deleted successfully. + +## Useful resources + +___ + +* [CloudFormation User Guide](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/index.html) +* [Kinesis Data Firehose User Guide](https://docs.aws.amazon.com/firehose/latest/dev/what-is-this-service.html) +* [IoT Core User Guide](https://docs.aws.amazon.com/iot/latest/developerguide/index.html) +* [AWS Glue User Guide](https://docs.aws.amazon.com/glue/latest/dg/what-is-glue.html) +* [Amazon Athena User Guide](https://docs.aws.amazon.com/athena/latest/ug/what-is.html) +* [AWS CDK (v2) User Guide](https://docs.aws.amazon.com/cdk/v2/guide/index.html) +* [Amazon Managed Grafana User Guide](https://docs.aws.amazon.com/grafana/latest/userguide/index.html)