From 3499c5c08959b24ee93b504c7af8282b0785e8f4 Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Fri, 12 Aug 2022 17:25:59 -0400 Subject: [PATCH 1/3] new: solr may take some time now :( With the new logic of waiting for the lock file to be deleted, the solr service might take some time to get ready... - The default timeout for a terraform resource is 20 mins.. - https://www.terraform.io/plugin/sdkv2/resources/retries-and-customizable-timeouts#default-timeouts-and-deadline-exceeded-errors - The default timeout for the ecs service resource is ~10 mins.. - https://github.com/hashicorp/terraform-provider-aws/issues/16012#issuecomment-732386175 We are setting the timeout to be a little higher; while also deregistering the old instance faster (allowing for a quicker turnaround between restarts); we need the 'wait_for_steady_state' in order to setup the admin username/password properly --- terraform/ecs/provision/follower.tf | 4 ++++ terraform/ecs/provision/leader.tf | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/terraform/ecs/provision/follower.tf b/terraform/ecs/provision/follower.tf index a78cd3f..fc7102d 100644 --- a/terraform/ecs/provision/follower.tf +++ b/terraform/ecs/provision/follower.tf @@ -115,6 +115,9 @@ resource "aws_ecs_task_definition" "solr-follower" { resource "aws_ecs_service" "solr-follower" { + timeouts { + create = "15m" + } count = var.solrFollowerCount name = "solr-follower-${count.index}-${var.instance_name}" cluster = aws_ecs_cluster.solr-cluster.id @@ -123,6 +126,7 @@ resource "aws_ecs_service" "solr-follower" { launch_type = "FARGATE" platform_version = "1.4.0" wait_for_steady_state = true + deregistration_delay = 90 network_configuration { security_groups = [module.vpc.default_security_group_id, aws_security_group.solr-ecs-efs-ingress.id] diff --git a/terraform/ecs/provision/leader.tf b/terraform/ecs/provision/leader.tf index 12d5d6c..6203065 100644 --- a/terraform/ecs/provision/leader.tf +++ b/terraform/ecs/provision/leader.tf @@ -143,6 +143,9 @@ resource "aws_ecs_task_definition" "solr-no-efs" { } resource "aws_ecs_service" "solr" { + timeouts { + create = "15m" + } name = "solr-${var.instance_name}" cluster = aws_ecs_cluster.solr-cluster.id task_definition = var.disableEfs ? aws_ecs_task_definition.solr-no-efs[0].arn : aws_ecs_task_definition.solr[0].arn @@ -150,6 +153,7 @@ resource "aws_ecs_service" "solr" { launch_type = "FARGATE" platform_version = "1.4.0" wait_for_steady_state = true + deregistration_delay = 90 network_configuration { security_groups = [module.vpc.default_security_group_id, aws_security_group.solr-ecs-efs-ingress.id] From 3f7e844fe1dd907d26214243839558bf94129134 Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Fri, 12 Aug 2022 17:35:24 -0400 Subject: [PATCH 2/3] fix: deregistration_delay is for target_group The target group needs to deregister the ecs task before it can be promptly restarted, so quicken that interval --- terraform/ecs/provision/follower.tf | 1 - terraform/ecs/provision/lb-follower.tf | 2 ++ terraform/ecs/provision/lb-leader.tf | 1 + terraform/ecs/provision/leader.tf | 1 - 4 files changed, 3 insertions(+), 2 deletions(-) diff --git a/terraform/ecs/provision/follower.tf b/terraform/ecs/provision/follower.tf index fc7102d..1dbecfa 100644 --- a/terraform/ecs/provision/follower.tf +++ b/terraform/ecs/provision/follower.tf @@ -126,7 +126,6 @@ resource "aws_ecs_service" "solr-follower" { launch_type = "FARGATE" platform_version = "1.4.0" wait_for_steady_state = true - deregistration_delay = 90 network_configuration { security_groups = [module.vpc.default_security_group_id, aws_security_group.solr-ecs-efs-ingress.id] diff --git a/terraform/ecs/provision/lb-follower.tf b/terraform/ecs/provision/lb-follower.tf index c4174f5..f103c05 100644 --- a/terraform/ecs/provision/lb-follower.tf +++ b/terraform/ecs/provision/lb-follower.tf @@ -61,6 +61,7 @@ resource "aws_lb_target_group" "solr-follower-target" { protocol = "HTTP" target_type = "ip" vpc_id = module.vpc.vpc_id + deregistration_delay = 90 health_check { healthy_threshold = 3 @@ -105,6 +106,7 @@ resource "aws_lb_target_group" "solr-follower-individual-target" { protocol = "HTTP" target_type = "ip" vpc_id = module.vpc.vpc_id + deregistration_delay = 90 health_check { healthy_threshold = 3 diff --git a/terraform/ecs/provision/lb-leader.tf b/terraform/ecs/provision/lb-leader.tf index 98f69aa..4a8bdd5 100644 --- a/terraform/ecs/provision/lb-leader.tf +++ b/terraform/ecs/provision/lb-leader.tf @@ -57,6 +57,7 @@ resource "aws_lb_target_group" "solr-target" { protocol = "HTTP" target_type = "ip" vpc_id = module.vpc.vpc_id + deregistration_delay = 90 health_check { healthy_threshold = 3 diff --git a/terraform/ecs/provision/leader.tf b/terraform/ecs/provision/leader.tf index 6203065..40b0fe3 100644 --- a/terraform/ecs/provision/leader.tf +++ b/terraform/ecs/provision/leader.tf @@ -153,7 +153,6 @@ resource "aws_ecs_service" "solr" { launch_type = "FARGATE" platform_version = "1.4.0" wait_for_steady_state = true - deregistration_delay = 90 network_configuration { security_groups = [module.vpc.default_security_group_id, aws_security_group.solr-ecs-efs-ingress.id] From c54d3f1a5105d2a341416420ef9391bde2ab1796 Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Fri, 12 Aug 2022 17:50:56 -0400 Subject: [PATCH 3/3] fix: timeouts are not configureable for this resource --- terraform/ecs/provision/follower.tf | 3 --- terraform/ecs/provision/leader.tf | 3 --- 2 files changed, 6 deletions(-) diff --git a/terraform/ecs/provision/follower.tf b/terraform/ecs/provision/follower.tf index 1dbecfa..a78cd3f 100644 --- a/terraform/ecs/provision/follower.tf +++ b/terraform/ecs/provision/follower.tf @@ -115,9 +115,6 @@ resource "aws_ecs_task_definition" "solr-follower" { resource "aws_ecs_service" "solr-follower" { - timeouts { - create = "15m" - } count = var.solrFollowerCount name = "solr-follower-${count.index}-${var.instance_name}" cluster = aws_ecs_cluster.solr-cluster.id diff --git a/terraform/ecs/provision/leader.tf b/terraform/ecs/provision/leader.tf index 40b0fe3..12d5d6c 100644 --- a/terraform/ecs/provision/leader.tf +++ b/terraform/ecs/provision/leader.tf @@ -143,9 +143,6 @@ resource "aws_ecs_task_definition" "solr-no-efs" { } resource "aws_ecs_service" "solr" { - timeouts { - create = "15m" - } name = "solr-${var.instance_name}" cluster = aws_ecs_cluster.solr-cluster.id task_definition = var.disableEfs ? aws_ecs_task_definition.solr-no-efs[0].arn : aws_ecs_task_definition.solr[0].arn