From 58f4d5c81d937d451c6a6d4f2193f7ffeca81403 Mon Sep 17 00:00:00 2001 From: "Matthew Bryant (mandatory)" Date: Tue, 7 Jan 2020 13:21:41 -0800 Subject: [PATCH] Added fix for ECS service-linked role (AWSServiceRoleForECS) issue --- api/server.py | 13 +- api/utils/aws_account_management/__init__.py | 0 .../aws_account_management/preterraform.py | 111 ++++++++++++++++++ 3 files changed, 123 insertions(+), 1 deletion(-) create mode 100644 api/utils/aws_account_management/__init__.py create mode 100644 api/utils/aws_account_management/preterraform.py diff --git a/api/server.py b/api/server.py index 14ce91820..0cbb82a10 100644 --- a/api/server.py +++ b/api/server.py @@ -57,6 +57,7 @@ from utils.deployments.awslambda import lambda_manager from utils.deployments.api_gateway import api_gateway_manager, strip_api_gateway from utils.deployments.shared_files import add_shared_files_to_zip, get_shared_files_for_lambda, add_shared_files_symlink_to_zip +from utils.aws_account_management.preterraform import preterraform_manager from services.websocket_router import WebSocketRouter, run_scheduled_heartbeat @@ -1185,6 +1186,11 @@ def terraform_apply( self, aws_account_data ): @staticmethod def _terraform_apply( aws_account_data ): + logit( "Ensuring existence of ECS service-linked role before continuing with terraform apply..." ) + preterraform_manager._ensure_ecs_service_linked_role_exists( + aws_account_data + ) + # The return data return_data = { "success": True, @@ -1303,6 +1309,11 @@ def _terraform_plan( aws_account_data ): @staticmethod def _terraform_configure_aws_account( aws_account_data ): + logit( "Ensuring existence of ECS service-linked role before continuing with AWS account configuration..." ) + preterraform_manager._ensure_ecs_service_linked_role_exists( + aws_account_data + ) + terraform_configuration_data = TaskSpawner._write_terraform_base_files( aws_account_data ) @@ -11173,7 +11184,7 @@ def get_lambda_callback_endpoint( tornado_config ): ) logit( "Lambda callback endpoint is " + LAMBDA_CALLBACK_ENDPOINT ) - + server.start() websocket_server.start() tornado.ioloop.IOLoop.current().start() \ No newline at end of file diff --git a/api/utils/aws_account_management/__init__.py b/api/utils/aws_account_management/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/api/utils/aws_account_management/preterraform.py b/api/utils/aws_account_management/preterraform.py new file mode 100644 index 000000000..088f8abf6 --- /dev/null +++ b/api/utils/aws_account_management/preterraform.py @@ -0,0 +1,111 @@ +import time +import tornado +import botocore + +from tornado.concurrent import run_on_executor, futures +from utils.aws_client import get_aws_client +from utils.general import logit +from tornado import gen + +from botocore.exceptions import ClientError + +class PreTerraformManager(object): + """ + There are some steps that need to be done pre-terraform because + terraform can not handle certain situations where specific AWS + resources do not exist. + + One such example is AWSServiceRoleForECS which is a service-linked + role for AWS ECS. It is normally automatically created when you attempt + to create a new ECS cluster. However, due to AWS's eventual-consistency + nature, terraform will choke on setting up the ECS resources because the + AWSServiceRoleForECS role will not be immediately available. Because of + this terraform will attempt to use it when it's not yet ready and will + choke out. + + The specific bug can be found here: + https://github.com/terraform-providers/terraform-provider-aws/issues/11417 + + To mitigate this, we use the Boto3 API to create the AWSServiceRoleForECS + role ahead of time before we run terraform. We then wait for the propogation + to finish before applying the actual terraform config. This mitigates the + problem from occuring. + """ + def __init__(self, loop=None): + self.executor = futures.ThreadPoolExecutor( 10 ) + self.loop = loop or tornado.ioloop.IOLoop.current() + + @run_on_executor + def ensure_ecs_service_linked_role_exists( self, credentials ): + return PreTerraformManager._ensure_ecs_service_linked_role_exists( credentials ) + + @staticmethod + def _ensure_ecs_service_linked_role_exists( credentials ): + logit( "Checking if ECS service-linked role exists..." ) + + # First check to see if the role exists + ecs_service_linked_role_exists = PreTerraformManager._check_if_ecs_service_linked_role_exists( + credentials + ) + + if ecs_service_linked_role_exists == True: + logit( "ECS service-linked role exists! We're good to go." ) + return + + logit( "ECS service-linked role does not exist, creating it..." ) + PreTerraformManager._create_ecs_service_linked_role( + credentials + ) + + logit( "ECS service-linked role created, waiting a bit before retrying operation..." ) + + # Wait a bit for propogation + time.sleep( 3 ) + + return PreTerraformManager._ensure_ecs_service_linked_role_exists( + credentials + ) + + @run_on_executor + def check_if_ecs_service_linked_role_exists( self, credentials ): + return PreTerraformManager._check_if_ecs_service_linked_role_exists( credentials ) + + @staticmethod + def _check_if_ecs_service_linked_role_exists( credentials ): + iam_client = get_aws_client( + "iam", + credentials + ) + + try: + linked_role_get_response = iam_client.get_role( + RoleName="AWSServiceRoleForECS" + ) + except botocore.exceptions.ClientError as boto_error: + if boto_error.response[ "Error" ][ "Code" ] != "NoSuchEntity": + # If it's not the exception we expect then throw + raise + return False + + return True + + @run_on_executor + def create_ecs_service_linked_role( self, credentials ): + return PreTerraformManager._create_ecs_service_linked_role( credentials ) + + @staticmethod + def _create_ecs_service_linked_role( credentials ): + iam_client = get_aws_client( + "iam", + credentials + ) + + created_service_linked_role_response = iam_client.create_service_linked_role( + AWSServiceName="ecs.amazonaws.com", + Description="Role to enable Amazon ECS to manage your cluster." + ) + + print( "Response from creating ECS service-linked role: " ) + print( created_service_linked_role_response ) + +preterraform_manager = PreTerraformManager() \ No newline at end of file