From 0f3d391bc2fabf53ef32d6bb04d1738e2ff2965e Mon Sep 17 00:00:00 2001 From: Samuel Date: Fri, 23 Jul 2021 11:43:16 +0200 Subject: [PATCH] fix(misc): contrib/terraform/aws (#7818) * fix(misc): terraform/aws - handles deployment with a single availability zone - handles deployment with more than two availability zone - handles etcd collocation with control-plane nodes (`aws_etcd_num=0`) - allows to set a bastion instances count (`aws_bastion_num`) - allows to set bastion/etcd/control-plane/workers rootfs volume size - removes variables from terraform.tfvars that were not re-used - adds .terraform.lock.hcl to .gitignore - changes/updates base image from ubuntu-18.03 to debian-10 tested by a few coworkers of mine, and myself: thanks for the outstanding work, on both those terraform samples and kubespray playbooks. I did not test ubuntu deployments, I could still swap from buster to focal. LMK. * fix(gitlab-ci) AFAIU, terraform.tfvars indentation should be fixed for / no diff returned running `terraform fmt -check -diff` https://gitlab.com/kargo-ci/kubernetes-sigs-kubespray/-/jobs/1445622114 --- contrib/terraform/aws/.gitignore | 1 + .../terraform/aws/create-infrastructure.tf | 30 +++++++++++----- contrib/terraform/aws/output.tf | 2 +- .../aws/sample-inventory/cluster.tfvars | 10 ++++-- contrib/terraform/aws/templates/inventory.tpl | 5 ++- contrib/terraform/aws/terraform.tfvars | 34 ++++++++++++------- .../terraform/aws/terraform.tfvars.example | 23 +++++++------ contrib/terraform/aws/variables.tf | 20 +++++++++-- 8 files changed, 84 insertions(+), 41 deletions(-) diff --git a/contrib/terraform/aws/.gitignore b/contrib/terraform/aws/.gitignore index 84fcb582197..373687b8014 100644 --- a/contrib/terraform/aws/.gitignore +++ b/contrib/terraform/aws/.gitignore @@ -1,2 +1,3 @@ *.tfstate* +.terraform.lock.hcl .terraform diff --git a/contrib/terraform/aws/create-infrastructure.tf b/contrib/terraform/aws/create-infrastructure.tf index 4a12ecb7440..f5ac5617b36 100644 --- a/contrib/terraform/aws/create-infrastructure.tf +++ b/contrib/terraform/aws/create-infrastructure.tf @@ -20,7 +20,7 @@ module "aws-vpc" { aws_cluster_name = var.aws_cluster_name aws_vpc_cidr_block = var.aws_vpc_cidr_block - aws_avail_zones = slice(data.aws_availability_zones.available.names, 0, 2) + aws_avail_zones = slice(data.aws_availability_zones.available.names, 0, length(var.aws_cidr_subnets_public) <= length(data.aws_availability_zones.available.names) ? length(var.aws_cidr_subnets_public) : length(data.aws_availability_zones.available.names)) aws_cidr_subnets_private = var.aws_cidr_subnets_private aws_cidr_subnets_public = var.aws_cidr_subnets_public default_tags = var.default_tags @@ -31,7 +31,7 @@ module "aws-elb" { aws_cluster_name = var.aws_cluster_name aws_vpc_id = module.aws-vpc.aws_vpc_id - aws_avail_zones = slice(data.aws_availability_zones.available.names, 0, 2) + aws_avail_zones = slice(data.aws_availability_zones.available.names, 0, length(var.aws_cidr_subnets_public) <= length(data.aws_availability_zones.available.names) ? length(var.aws_cidr_subnets_public) : length(data.aws_availability_zones.available.names)) aws_subnet_ids_public = module.aws-vpc.aws_subnet_ids_public aws_elb_api_port = var.aws_elb_api_port k8s_secure_api_port = var.k8s_secure_api_port @@ -52,9 +52,9 @@ module "aws-iam" { resource "aws_instance" "bastion-server" { ami = data.aws_ami.distro.id instance_type = var.aws_bastion_size - count = length(var.aws_cidr_subnets_public) + count = var.aws_bastion_num associate_public_ip_address = true - availability_zone = element(slice(data.aws_availability_zones.available.names, 0, 2), count.index) + availability_zone = element(slice(data.aws_availability_zones.available.names, 0, length(var.aws_cidr_subnets_public) <= length(data.aws_availability_zones.available.names) ? length(var.aws_cidr_subnets_public) : length(data.aws_availability_zones.available.names)), count.index) subnet_id = element(module.aws-vpc.aws_subnet_ids_public, count.index) vpc_security_group_ids = module.aws-vpc.aws_security_group @@ -79,11 +79,15 @@ resource "aws_instance" "k8s-master" { count = var.aws_kube_master_num - availability_zone = element(slice(data.aws_availability_zones.available.names, 0, 2), count.index) + availability_zone = element(slice(data.aws_availability_zones.available.names, 0, length(var.aws_cidr_subnets_public) <= length(data.aws_availability_zones.available.names) ? length(var.aws_cidr_subnets_public) : length(data.aws_availability_zones.available.names)), count.index) subnet_id = element(module.aws-vpc.aws_subnet_ids_private, count.index) vpc_security_group_ids = module.aws-vpc.aws_security_group + root_block_device { + volume_size = var.aws_kube_master_disk_size + } + iam_instance_profile = module.aws-iam.kube_control_plane-profile key_name = var.AWS_SSH_KEY_NAME @@ -106,11 +110,15 @@ resource "aws_instance" "k8s-etcd" { count = var.aws_etcd_num - availability_zone = element(slice(data.aws_availability_zones.available.names, 0, 2), count.index) + availability_zone = element(slice(data.aws_availability_zones.available.names, 0, length(var.aws_cidr_subnets_public) <= length(data.aws_availability_zones.available.names) ? length(var.aws_cidr_subnets_public) : length(data.aws_availability_zones.available.names)), count.index) subnet_id = element(module.aws-vpc.aws_subnet_ids_private, count.index) vpc_security_group_ids = module.aws-vpc.aws_security_group + root_block_device { + volume_size = var.aws_etcd_disk_size + } + key_name = var.AWS_SSH_KEY_NAME tags = merge(var.default_tags, tomap({ @@ -126,11 +134,15 @@ resource "aws_instance" "k8s-worker" { count = var.aws_kube_worker_num - availability_zone = element(slice(data.aws_availability_zones.available.names, 0, 2), count.index) + availability_zone = element(slice(data.aws_availability_zones.available.names, 0, length(var.aws_cidr_subnets_public) <= length(data.aws_availability_zones.available.names) ? length(var.aws_cidr_subnets_public) : length(data.aws_availability_zones.available.names)), count.index) subnet_id = element(module.aws-vpc.aws_subnet_ids_private, count.index) vpc_security_group_ids = module.aws-vpc.aws_security_group + root_block_device { + volume_size = var.aws_kube_worker_disk_size + } + iam_instance_profile = module.aws-iam.kube-worker-profile key_name = var.AWS_SSH_KEY_NAME @@ -152,10 +164,10 @@ data "template_file" "inventory" { public_ip_address_bastion = join("\n", formatlist("bastion ansible_host=%s", aws_instance.bastion-server.*.public_ip)) connection_strings_master = join("\n", formatlist("%s ansible_host=%s", aws_instance.k8s-master.*.private_dns, aws_instance.k8s-master.*.private_ip)) connection_strings_node = join("\n", formatlist("%s ansible_host=%s", aws_instance.k8s-worker.*.private_dns, aws_instance.k8s-worker.*.private_ip)) - connection_strings_etcd = join("\n", formatlist("%s ansible_host=%s", aws_instance.k8s-etcd.*.private_dns, aws_instance.k8s-etcd.*.private_ip)) list_master = join("\n", aws_instance.k8s-master.*.private_dns) list_node = join("\n", aws_instance.k8s-worker.*.private_dns) - list_etcd = join("\n", aws_instance.k8s-etcd.*.private_dns) + connection_strings_etcd = join("\n", formatlist("%s ansible_host=%s", ((var.aws_etcd_num > 0) ? (aws_instance.k8s-etcd.*.private_dns) : (aws_instance.k8s-master.*.private_dns)), ((var.aws_etcd_num > 0) ? (aws_instance.k8s-etcd.*.private_ip) : (aws_instance.k8s-master.*.private_ip)))) + list_etcd = join("\n", ((var.aws_etcd_num > 0) ? (aws_instance.k8s-etcd.*.private_dns) : (aws_instance.k8s-master.*.private_dns))) elb_api_fqdn = "apiserver_loadbalancer_domain_name=\"${module.aws-elb.aws_elb_api_fqdn}\"" } } diff --git a/contrib/terraform/aws/output.tf b/contrib/terraform/aws/output.tf index 35e6e27a72f..8cac230af25 100644 --- a/contrib/terraform/aws/output.tf +++ b/contrib/terraform/aws/output.tf @@ -11,7 +11,7 @@ output "workers" { } output "etcd" { - value = join("\n", aws_instance.k8s-etcd.*.private_ip) + value = join("\n", ((var.aws_etcd_num > 0) ? (aws_instance.k8s-etcd.*.private_ip) : (aws_instance.k8s-master.*.private_ip))) } output "aws_elb_api_fqdn" { diff --git a/contrib/terraform/aws/sample-inventory/cluster.tfvars b/contrib/terraform/aws/sample-inventory/cluster.tfvars index 1b0b76c9787..d731a041652 100644 --- a/contrib/terraform/aws/sample-inventory/cluster.tfvars +++ b/contrib/terraform/aws/sample-inventory/cluster.tfvars @@ -9,6 +9,8 @@ aws_cidr_subnets_private = ["10.250.192.0/20", "10.250.208.0/20"] aws_cidr_subnets_public = ["10.250.224.0/20", "10.250.240.0/20"] #Bastion Host +aws_bastion_num = 1 + aws_bastion_size = "t2.medium" #Kubernetes Cluster @@ -17,22 +19,26 @@ aws_kube_master_num = 3 aws_kube_master_size = "t2.medium" +aws_kube_master_disk_size = 50 + aws_etcd_num = 3 aws_etcd_size = "t2.medium" +aws_etcd_disk_size = 50 + aws_kube_worker_num = 4 aws_kube_worker_size = "t2.medium" +aws_kube_worker_disk_size = 50 + #Settings AWS ELB aws_elb_api_port = 6443 k8s_secure_api_port = 6443 -kube_insecure_apiserver_address = "0.0.0.0" - default_tags = { # Env = "devtest" # Product = "kubernetes" } diff --git a/contrib/terraform/aws/templates/inventory.tpl b/contrib/terraform/aws/templates/inventory.tpl index baa9ea8545c..c0d0d10244c 100644 --- a/contrib/terraform/aws/templates/inventory.tpl +++ b/contrib/terraform/aws/templates/inventory.tpl @@ -10,19 +10,18 @@ ${public_ip_address_bastion} [kube_control_plane] ${list_master} - [kube_node] ${list_node} - [etcd] ${list_etcd} +[calico_rr] [k8s_cluster:children] kube_node kube_control_plane - +calico_rr [k8s_cluster:vars] ${elb_api_fqdn} diff --git a/contrib/terraform/aws/terraform.tfvars b/contrib/terraform/aws/terraform.tfvars index c8db6b424a2..21089ebdd1a 100644 --- a/contrib/terraform/aws/terraform.tfvars +++ b/contrib/terraform/aws/terraform.tfvars @@ -6,26 +6,34 @@ aws_vpc_cidr_block = "10.250.192.0/18" aws_cidr_subnets_private = ["10.250.192.0/20", "10.250.208.0/20"] aws_cidr_subnets_public = ["10.250.224.0/20", "10.250.240.0/20"] -#Bastion Host -aws_bastion_size = "t2.medium" +# single AZ deployment +#aws_cidr_subnets_private = ["10.250.192.0/20"] +#aws_cidr_subnets_public = ["10.250.224.0/20"] +# 3+ AZ deployment +#aws_cidr_subnets_private = ["10.250.192.0/24","10.250.193.0/24","10.250.194.0/24","10.250.195.0/24"] +#aws_cidr_subnets_public = ["10.250.224.0/24","10.250.225.0/24","10.250.226.0/24","10.250.227.0/24"] -#Kubernetes Cluster +#Bastion Host +aws_bastion_num = 1 +aws_bastion_size = "t3.small" -aws_kube_master_num = 3 -aws_kube_master_size = "t2.medium" +#Kubernetes Cluster +aws_kube_master_num = 3 +aws_kube_master_size = "t3.medium" +aws_kube_master_disk_size = 50 -aws_etcd_num = 3 -aws_etcd_size = "t2.medium" +aws_etcd_num = 0 +aws_etcd_size = "t3.medium" +aws_etcd_disk_size = 50 -aws_kube_worker_num = 4 -aws_kube_worker_size = "t2.medium" +aws_kube_worker_num = 4 +aws_kube_worker_size = "t3.medium" +aws_kube_worker_disk_size = 50 #Settings AWS ELB - -aws_elb_api_port = 6443 -k8s_secure_api_port = 6443 -kube_insecure_apiserver_address = "0.0.0.0" +aws_elb_api_port = 6443 +k8s_secure_api_port = 6443 default_tags = { # Env = "devtest" diff --git a/contrib/terraform/aws/terraform.tfvars.example b/contrib/terraform/aws/terraform.tfvars.example index 666b21db226..76684d831f1 100644 --- a/contrib/terraform/aws/terraform.tfvars.example +++ b/contrib/terraform/aws/terraform.tfvars.example @@ -8,25 +8,26 @@ aws_cidr_subnets_public = ["10.250.224.0/20","10.250.240.0/20"] aws_avail_zones = ["eu-central-1a","eu-central-1b"] #Bastion Host -aws_bastion_ami = "ami-5900cc36" -aws_bastion_size = "t2.small" - +aws_bastion_num = 1 +aws_bastion_size = "t3.small" #Kubernetes Cluster - aws_kube_master_num = 3 -aws_kube_master_size = "t2.medium" +aws_kube_master_size = "t3.medium" +aws_kube_master_disk_size = 50 aws_etcd_num = 3 -aws_etcd_size = "t2.medium" +aws_etcd_size = "t3.medium" +aws_etcd_disk_size = 50 aws_kube_worker_num = 4 -aws_kube_worker_size = "t2.medium" - -aws_cluster_ami = "ami-903df7ff" +aws_kube_worker_size = "t3.medium" +aws_kube_worker_disk_size = 50 #Settings AWS ELB - aws_elb_api_port = 6443 k8s_secure_api_port = 6443 -kube_insecure_apiserver_address = 0.0.0.0 + +default_tags = { } + +inventory_file = "../../../inventory/hosts" diff --git a/contrib/terraform/aws/variables.tf b/contrib/terraform/aws/variables.tf index 19d165f04c3..92a5512c838 100644 --- a/contrib/terraform/aws/variables.tf +++ b/contrib/terraform/aws/variables.tf @@ -25,7 +25,7 @@ data "aws_ami" "distro" { filter { name = "name" - values = ["ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-*"] + values = ["debian-10-amd64-*"] } filter { @@ -33,7 +33,7 @@ data "aws_ami" "distro" { values = ["hvm"] } - owners = ["099720109477"] # Canonical + owners = ["136693071363"] # Debian-10 } //AWS VPC Variables @@ -63,10 +63,18 @@ variable "aws_bastion_size" { * The number should be divisable by the number of used * AWS Availability Zones without an remainder. */ +variable "aws_bastion_num" { + description = "Number of Bastion Nodes" +} + variable "aws_kube_master_num" { description = "Number of Kubernetes Master Nodes" } +variable "aws_kube_master_disk_size" { + description = "Disk size for Kubernetes Master Nodes (in GiB)" +} + variable "aws_kube_master_size" { description = "Instance size of Kube Master Nodes" } @@ -75,6 +83,10 @@ variable "aws_etcd_num" { description = "Number of etcd Nodes" } +variable "aws_etcd_disk_size" { + description = "Disk size for etcd Nodes (in GiB)" +} + variable "aws_etcd_size" { description = "Instance size of etcd Nodes" } @@ -83,6 +95,10 @@ variable "aws_kube_worker_num" { description = "Number of Kubernetes Worker Nodes" } +variable "aws_kube_worker_disk_size" { + description = "Disk size for Kubernetes Worker Nodes (in GiB)" +} + variable "aws_kube_worker_size" { description = "Instance size of Kubernetes Worker Nodes" }