Added fix for ECS service-linked role (AWSServiceRoleForECS) issue

api/server.py

@@ -57,6 +57,7 @@
 from utils.deployments.awslambda import lambda_manager
 from utils.deployments.api_gateway import api_gateway_manager, strip_api_gateway
 from utils.deployments.shared_files import add_shared_files_to_zip, get_shared_files_for_lambda, add_shared_files_symlink_to_zip
+from utils.aws_account_management.preterraform import preterraform_manager
 
 from services.websocket_router import WebSocketRouter, run_scheduled_heartbeat
 
@@ -1185,6 +1186,11 @@ def terraform_apply( self, aws_account_data ):
 
 		@staticmethod
 		def _terraform_apply( aws_account_data ):
+			logit( "Ensuring existence of ECS service-linked role before continuing with terraform apply..." )
+			preterraform_manager._ensure_ecs_service_linked_role_exists(
+				aws_account_data
+			)
+
 			# The return data
 			return_data = {
 				"success": True,
@@ -1303,6 +1309,11 @@ def _terraform_plan( aws_account_data ):
 
 		@staticmethod
 		def _terraform_configure_aws_account( aws_account_data ):
+			logit( "Ensuring existence of ECS service-linked role before continuing with AWS account configuration..." )
+			preterraform_manager._ensure_ecs_service_linked_role_exists(
+				aws_account_data
+			)
+
 			terraform_configuration_data = TaskSpawner._write_terraform_base_files(
 				aws_account_data
 			)
@@ -11173,7 +11184,7 @@ def get_lambda_callback_endpoint( tornado_config ):
 	)
 
 	logit( "Lambda callback endpoint is " + LAMBDA_CALLBACK_ENDPOINT )
-		
+
 	server.start()
 	websocket_server.start()
 	tornado.ioloop.IOLoop.current().start()

api/utils/aws_account_management/preterraform.py

@@ -0,0 +1,111 @@
+import time
+import tornado
+import botocore
+
+from tornado.concurrent import run_on_executor, futures
+from utils.aws_client import get_aws_client
+from utils.general import logit
+from tornado import gen
+
+from botocore.exceptions import ClientError
+
+class PreTerraformManager(object):
+	"""
+	There are some steps that need to be done pre-terraform because
+	terraform can not handle certain situations where specific AWS
+	resources do not exist.
+
+	One such example is AWSServiceRoleForECS which is a service-linked
+	role for AWS ECS. It is normally automatically created when you attempt
+	to create a new ECS cluster. However, due to AWS's eventual-consistency
+	nature, terraform will choke on setting up the ECS resources because the
+	AWSServiceRoleForECS role will not be immediately available. Because of 
+	this terraform will attempt to use it when it's not yet ready and will
+	choke out.
+
+	The specific bug can be found here:
+	https://github.com/terraform-providers/terraform-provider-aws/issues/11417
+
+	To mitigate this, we use the Boto3 API to create the AWSServiceRoleForECS
+	role ahead of time before we run terraform. We then wait for the propogation
+	to finish before applying the actual terraform config. This mitigates the
+	problem from occuring.
+	"""
+	def __init__(self, loop=None):
+		self.executor = futures.ThreadPoolExecutor( 10 )
+		self.loop = loop or tornado.ioloop.IOLoop.current()
+
+	@run_on_executor
+	def ensure_ecs_service_linked_role_exists( self, credentials ):
+		return PreTerraformManager._ensure_ecs_service_linked_role_exists( credentials )
+
+	@staticmethod
+	def _ensure_ecs_service_linked_role_exists( credentials ):
+		logit( "Checking if ECS service-linked role exists..." )
+
+		# First check to see if the role exists
+		ecs_service_linked_role_exists = PreTerraformManager._check_if_ecs_service_linked_role_exists(
+			credentials
+		)
+
+		if ecs_service_linked_role_exists == True:
+			logit( "ECS service-linked role exists! We're good to go." )
+			return
+
+		logit( "ECS service-linked role does not exist, creating it..." )
+		PreTerraformManager._create_ecs_service_linked_role(
+			credentials
+		)
+
+		logit( "ECS service-linked role created, waiting a bit before retrying operation..." )
+
+		# Wait a bit for propogation
+		time.sleep( 3 )
+
+		return PreTerraformManager._ensure_ecs_service_linked_role_exists(
+			credentials
+		)
+
+	@run_on_executor
+	def check_if_ecs_service_linked_role_exists( self, credentials ):
+		return PreTerraformManager._check_if_ecs_service_linked_role_exists( credentials )
+
+	@staticmethod
+	def _check_if_ecs_service_linked_role_exists( credentials ):
+		iam_client = get_aws_client(
+			"iam",
+			credentials
+		)
+
+		try:
+			linked_role_get_response = iam_client.get_role(
+				RoleName="AWSServiceRoleForECS"
+			)
+		except botocore.exceptions.ClientError as boto_error:
+			if boto_error.response[ "Error" ][ "Code" ] != "NoSuchEntity":
+				# If it's not the exception we expect then throw
+				raise
+			return False
+
+		return True
+
+	@run_on_executor
+	def create_ecs_service_linked_role( self, credentials ):
+		return PreTerraformManager._create_ecs_service_linked_role( credentials )
+
+	@staticmethod
+	def _create_ecs_service_linked_role( credentials ):
+		iam_client = get_aws_client(
+			"iam",
+			credentials
+		)
+
+		created_service_linked_role_response = iam_client.create_service_linked_role(
+			AWSServiceName="ecs.amazonaws.com",
+			Description="Role to enable Amazon ECS to manage your cluster."
+		)
+
+		print( "Response from creating ECS service-linked role: " )
+		print( created_service_linked_role_response )
+
+preterraform_manager = PreTerraformManager()