diff --git a/.gitignore b/.gitignore index 3e73ef9c..80a7ae41 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,6 @@ hugo.linux # Temporary lock file while building /.hugo_build.lock **/nohup.out + +.terraform* +terraform.tfstate* \ No newline at end of file diff --git a/content/using_ec2_spot_instances_with_eks/010_prerequisites/prerequisites.files/eks-blueprints.tf b/content/using_ec2_spot_instances_with_eks/010_prerequisites/prerequisites.files/eks-blueprints.tf index b7790d03..47419fe0 100644 --- a/content/using_ec2_spot_instances_with_eks/010_prerequisites/prerequisites.files/eks-blueprints.tf +++ b/content/using_ec2_spot_instances_with_eks/010_prerequisites/prerequisites.files/eks-blueprints.tf @@ -22,21 +22,21 @@ provider "aws" { } provider "kubernetes" { - host = module.eks_blueprints.eks_cluster_endpoint - cluster_ca_certificate = base64decode(module.eks_blueprints.eks_cluster_certificate_authority_data) + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) token = data.aws_eks_cluster_auth.this.token } provider "helm" { kubernetes { - host = module.eks_blueprints.eks_cluster_endpoint - cluster_ca_certificate = base64decode(module.eks_blueprints.eks_cluster_certificate_authority_data) + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) token = data.aws_eks_cluster_auth.this.token } } data "aws_eks_cluster_auth" "this" { - name = module.eks_blueprints.eks_cluster_id + name = module.eks.cluster_name } data "aws_ami" "amazonlinux2eks" { @@ -66,68 +66,65 @@ locals { } } -#--------------------------------------------------------------- -# EKS Blueprints -#--------------------------------------------------------------- -module "eks_blueprints" { - source = "github.com/aws-ia/terraform-aws-eks-blueprints?ref=v4.21.0" - - cluster_name = local.name - cluster_version = local.cluster_version - - vpc_id = module.vpc.vpc_id - private_subnet_ids = module.vpc.private_subnets - - node_security_group_additional_rules = { - # Extend node-to-node security group rules. Recommended and required for the Add-ons - ingress_self_all = { - description = "Node to node all ports/protocols" - protocol = "-1" - from_port = 0 - to_port = 0 - type = "ingress" - self = true - } - # Recommended outbound traffic for Node groups - egress_all = { - description = "Node all egress" - protocol = "-1" - from_port = 0 - to_port = 0 - type = "egress" - cidr_blocks = ["0.0.0.0/0"] - ipv6_cidr_blocks = ["::/0"] - } - # Allows Control Plane Nodes to talk to Worker nodes on all ports. Added this to simplify the example and further avoid issues with Add-ons communication with Control plane. - # This can be restricted further to specific port based on the requirement for each Add-on e.g., metrics-server 4443, spark-operator 8080, karpenter 8443 etc. - # Change this according to your security requirements if needed - ingress_cluster_to_node_all_traffic = { - description = "Cluster API to Nodegroup all traffic" - protocol = "-1" - from_port = 0 - to_port = 0 - type = "ingress" - source_cluster_security_group = true +################################################################################ +# Cluster +################################################################################ +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "19.16.0" + + cluster_name = local.name + cluster_version = local.cluster_version + cluster_endpoint_public_access = true + + cluster_addons = { + # aws-ebs-csi-driver = { most_recent = true } + kube-proxy = { most_recent = true } + coredns = { most_recent = true } + + vpc-cni = { + most_recent = true + before_compute = true + configuration_values = jsonencode({ + env = { + ENABLE_PREFIX_DELEGATION = "true" + WARM_PREFIX_TARGET = "1" + } + }) } } - managed_node_groups = { - # Managed Node groups with minimum config - mg5 = { + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + create_cloudwatch_log_group = false + + manage_aws_auth_configmap = true + + eks_managed_node_groups = { + mg_5 = { node_group_name = "mg5" instance_types = ["m4.xlarge", "m5.xlarge", "m5a.xlarge", "m5ad.xlarge", "m5d.xlarge", "t2.xlarge", "t3.xlarge", "t3a.xlarge"] - min_size = 2 + + create_security_group = false + + subnet_ids = module.vpc.private_subnets + max_size = 2 + desired_size = 2 + min_size = 2 + create_iam_role = false iam_role_arn = aws_iam_role.managed_ng.arn disk_size = 100 - update_config = [{ - max_unavailable_percentage = 30 - }] - k8s_labels = { + # Launch template configuration + create_launch_template = true # false will use the default launch template + launch_template_os = "amazonlinux2eks" # amazonlinux2eks or bottlerocket` + + labels = { intent = "control-apps" } - }, + } // ### -->> SPOT NODE GROUPS GO HERE <<--- ### } @@ -137,21 +134,20 @@ module "eks_blueprints" { tags = local.tags } -module "eks_blueprints_kubernetes_addons" { - source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons?ref=v4.21.0" +module "eks_blueprints_addons" { + source = "aws-ia/eks-blueprints-addons/aws" + version = "1.7.0" - eks_cluster_id = module.eks_blueprints.eks_cluster_id - eks_cluster_endpoint = module.eks_blueprints.eks_cluster_endpoint - eks_oidc_provider = module.eks_blueprints.oidc_provider - eks_cluster_version = module.eks_blueprints.eks_cluster_version + cluster_name = module.eks.cluster_name + cluster_endpoint = module.eks.cluster_endpoint + cluster_version = module.eks.cluster_version + oidc_provider_arn = module.eks.oidc_provider_arn + + create_delay_dependencies = [for prof in module.eks.eks_managed_node_groups : prof.node_group_arn] enable_metrics_server = true tags = local.tags - - depends_on = [ - module.eks_blueprints - ] } #--------------------------------------------------------------- @@ -159,7 +155,7 @@ module "eks_blueprints_kubernetes_addons" { #--------------------------------------------------------------- module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "4.0.2" + version = "5.0.0" name = local.name cidr = local.vpc_cidr @@ -240,5 +236,5 @@ resource "aws_iam_instance_profile" "managed_ng" { output "configure_kubectl" { description = "Configure kubectl: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig" - value = module.eks_blueprints.configure_kubectl -} + value = "aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name}" +} \ No newline at end of file diff --git a/content/using_ec2_spot_instances_with_eks/010_prerequisites/prerequisites.files/eks-spot-workshop-quickstart-cnf.yml b/content/using_ec2_spot_instances_with_eks/010_prerequisites/prerequisites.files/eks-spot-workshop-quickstart-cnf.yml index c51fed5b..c65d9099 100644 --- a/content/using_ec2_spot_instances_with_eks/010_prerequisites/prerequisites.files/eks-spot-workshop-quickstart-cnf.yml +++ b/content/using_ec2_spot_instances_with_eks/010_prerequisites/prerequisites.files/eks-spot-workshop-quickstart-cnf.yml @@ -28,22 +28,22 @@ Parameters: C9KubectlVersion: Description: Cloud9 instance kubectl version Type: String - Default: v1.23.7 + Default: v1.27.3 ConstraintDescription: Must be a valid kubectl version C9KubectlVersionTEST: Description: Cloud9 instance kubectl version Type: String - Default: v1.23.7 + Default: v1.27.3 ConstraintDescription: Must be a valid kubectl version C9EKSctlVersion: Description: Cloud9 instance eksctl version Type: String - Default: v0.110.0 + Default: v0.153.0 ConstraintDescription: Must be a valid eksctl version EKSClusterVersion: Description: EKS Cluster Version Type: String - Default: 1.24 + Default: 1.27 ConstraintDescription: Must be a valid eks version EKSClusterName: Description: EKS Cluster Name @@ -272,7 +272,7 @@ Resources: - echo '=== Install Terraform ===' - sudo yum install -y yum-utils - sudo yum-config-manager --add-repo https://rpm.releases.hashicorp.com/AmazonLinux/hashicorp.repo - - sudo yum -y install terraform-1.3.7-1 + - sudo yum -y install terraform-1.5.5 - echo '=== Create EKS Cluster ===' - sudo -H -u ec2-user mkdir -p /home/ec2-user/environment/eksworkshop/ - sudo -H -u ec2-user curl --silent --location -o /home/ec2-user/environment/eksworkshop/main.tf "https://raw.githubusercontent.com/awslabs/ec2-spot-workshops/master/content/using_ec2_spot_instances_with_eks/010_prerequisites/prerequisites.files/eks-blueprints.tf" diff --git a/content/using_ec2_spot_instances_with_eks/040_eksmanagednodegroupswithspot/workers_terraform.md b/content/using_ec2_spot_instances_with_eks/040_eksmanagednodegroupswithspot/workers_terraform.md index 062c41f7..9d659f15 100644 --- a/content/using_ec2_spot_instances_with_eks/040_eksmanagednodegroupswithspot/workers_terraform.md +++ b/content/using_ec2_spot_instances_with_eks/040_eksmanagednodegroupswithspot/workers_terraform.md @@ -27,11 +27,17 @@ Now copy the following code snippet, and paste it just below the `// ### -->> SP desired_size = 2 min_size = 0 - subnet_type = "private" - subnet_ids = [] + subnet_ids = module.vpc.private_subnets + + taints = { + spotInstance = { + key = "spotInstance" + value = "true" + effect = "PREFER_NO_SCHEDULE" + } + } - k8s_taints = [{ key = "spotInstance", value = "true", effect = "PREFER_NO_SCHEDULE" }] - k8s_labels = { + labels = { intent = "apps" } }, @@ -44,20 +50,22 @@ Now copy the following code snippet, and paste it just below the `// ### -->> SP desired_size = 1 min_size = 0 - subnet_type = "private" - subnet_ids = [] + subnet_ids = module.vpc.private_subnets + + taints = { + spotInstance = { + key = "spotInstance" + value = "true" + effect = "PREFER_NO_SCHEDULE" + } + } - k8s_taints = [{ key = "spotInstance", value = "true", effect = "PREFER_NO_SCHEDULE" }] - k8s_labels = { + labels = { intent = "apps" } } ``` -Now your `main.tf` file should look like this: - -![EKS Blueprints - Spot Node Groups Block](/images/using_ec2_spot_instances_with_eks/prerequisites/eksblueprints_spot_nodegroups_final.png) - Run the following command to fix any identation or configuration problem (if any): ``` diff --git a/content/using_ec2_spot_instances_with_eks/070_selfmanagednodegroupswithspot/workers_terraform.md b/content/using_ec2_spot_instances_with_eks/070_selfmanagednodegroupswithspot/workers_terraform.md index 3b62bb18..533091ca 100644 --- a/content/using_ec2_spot_instances_with_eks/070_selfmanagednodegroupswithspot/workers_terraform.md +++ b/content/using_ec2_spot_instances_with_eks/070_selfmanagednodegroupswithspot/workers_terraform.md @@ -34,43 +34,77 @@ Just below that line, paste the following code snippet to create two Spot self-m ``` self_managed_node_groups = { smng_spot_4vcpu_16mem = { - node_group_name = "smng-spot-4vcpu-16mem" - capacity_type = "spot" - capacity_rebalance = true - instance_types = ["m4.xlarge", "m5.xlarge", "m5a.xlarge", "m5ad.xlarge", "m5d.xlarge", "t2.xlarge", "t3.xlarge", "t3a.xlarge"] - - max_size = 4 - desired_size = 2 - min_size = 0 + node_group_name = "smng-spot-4vcpu-16mem" + capacity_rebalance = true + use_mixed_instances_policy = true + create_iam_role = false + iam_role_arn = aws_iam_role.managed_ng.arn + instance_type = "m5.xlarge" + + bootstrap_extra_args = "--kubelet-extra-args '--node-labels=eks.amazonaws.com/capacityType=SPOT,intent=apps,type=self-managed-spot --register-with-taints=spotInstance=true:PreferNoSchedule'" + + mixed_instances_policy = { + instances_distribution = { + on_demand_base_capacity = 0 + on_demand_percentage_above_base_capacity = 0 + spot_allocation_strategy = "price-capacity-optimized" + } + + override = [ + { instance_type = "m4.xlarge" }, + { instance_type = "m5.xlarge" }, + { instance_type = "m5a.xlarge" }, + { instance_type = "m5ad.xlarge" }, + { instance_type = "m5d.xlarge" }, + { instance_type = "t2.xlarge" }, + { instance_type = "t3.xlarge" }, + { instance_type = "t3a.xlarge" } + ] + } + + max_size = 4 + desired_size = 2 + min_size = 0 subnet_ids = module.vpc.private_subnets launch_template_os = "amazonlinux2eks" - - k8s_taints = [{ key = "spotInstance", value = "true", effect = "PREFER_NO_SCHEDULE" }] - k8s_labels = { - intent = "apps" - type = "self-managed-spot" - } } smng_spot_8vcpu_32mem = { - node_group_name = "smng-spot-8vcpu-32mem" - capacity_type = "spot" - capacity_rebalance = true - instance_types = ["m4.2xlarge", "m5.2xlarge", "m5a.2xlarge", "m5ad.2xlarge", "m5d.2xlarge", "t2.2xlarge", "t3.2xlarge", "t3a.2xlarge"] - - max_size = 2 - desired_size = 1 - min_size = 0 + node_group_name = "smng-spot-8vcpu-32mem" + capacity_rebalance = true + use_mixed_instances_policy = true + create_iam_role = false + iam_role_arn = aws_iam_role.managed_ng.arn + instance_type = "m5.2xlarge" + + bootstrap_extra_args = "--kubelet-extra-args '--node-labels=eks.amazonaws.com/capacityType=SPOT,intent=apps,type=self-managed-spot --register-with-taints=spotInstance=true:PreferNoSchedule'" + + mixed_instances_policy = { + instances_distribution = { + on_demand_base_capacity = 0 + on_demand_percentage_above_base_capacity = 0 + spot_allocation_strategy = "price-capacity-optimized" + } + + override = [ + { instance_type = "m4.2xlarge" }, + { instance_type = "m5.2xlarge" }, + { instance_type = "m5a.2xlarge" }, + { instance_type = "m5ad.2xlarge" }, + { instance_type = "m5d.2xlarge" }, + { instance_type = "t2.2xlarge" }, + { instance_type = "t3.2xlarge" }, + { instance_type = "t3a.2xlarge" } + ] + } - subnet_ids = module.vpc.private_subnets - launch_template_os = "amazonlinux2eks" + max_size = 2 + desired_size = 1 + min_size = 0 - k8s_taints = [{ key = "spotInstance", value = "true", effect = "PREFER_NO_SCHEDULE" }] - k8s_labels = { - intent = "apps" - type = "self-managed-spot" - } + subnet_ids = module.vpc.private_subnets + launch_template_os = "amazonlinux2eks" } } ``` diff --git a/static/images/using_ec2_spot_instances_with_eks/prerequisites/eksblueprints_clusterautoscaler.png b/static/images/using_ec2_spot_instances_with_eks/prerequisites/eksblueprints_clusterautoscaler.png index 2ffe14e4..927cb36a 100644 Binary files a/static/images/using_ec2_spot_instances_with_eks/prerequisites/eksblueprints_clusterautoscaler.png and b/static/images/using_ec2_spot_instances_with_eks/prerequisites/eksblueprints_clusterautoscaler.png differ diff --git a/static/images/using_ec2_spot_instances_with_eks/prerequisites/eksblueprints_spot_nodegroups.png b/static/images/using_ec2_spot_instances_with_eks/prerequisites/eksblueprints_spot_nodegroups.png index 489cdeb9..00ff56b1 100644 Binary files a/static/images/using_ec2_spot_instances_with_eks/prerequisites/eksblueprints_spot_nodegroups.png and b/static/images/using_ec2_spot_instances_with_eks/prerequisites/eksblueprints_spot_nodegroups.png differ