Automation to assign a Elastic IP to a Cluster and perform a reboot test

Signed-off-by: est-suse <[email protected]>

modules/install/install_rke2_master_eip.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+# This script installs the first master, ensuring first master is installed
+# and ready before proceeding to install other nodes
+set -x
+echo "$@"
+
+create_lb=$1
+public_ip=$2
+server_flags=$3
+
+hostname=$(hostname -f)
+mkdir -p /etc/rancher/rke2
+cat << EOF >/etc/rancher/rke2/config.yaml
+write-kubeconfig-mode: "0644"
+tls-san:
+  - ${create_lb}
+node-name: ${hostname}
+EOF
+
+if [ -n "$server_flags" ] && [[ "$server_flags" == *":"* ]]
+then
+   echo "$server_flags"
+   echo -e "$server_flags" >> /etc/rancher/rke2/config.yaml
+   if [[ "$server_flags" != *"cloud-provider-name"* ]]
+   then
+     echo -e "node-external-ip: $public_ip" >> /etc/rancher/rke2/config.yaml
+   fi
+   cat /etc/rancher/rke2/config.yaml
+else
+  echo -e "node-external-ip: $public_ip" >> /etc/rancher/rke2/config.yaml
+fi
+
+sudo systemctl restart rke2-server
+

modules/install/join_rke2_agent_eip.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+# This script is used to join one or more nodes as agents
+set -x
+echo "$@"
+
+server_ip=$1
+token=$2
+public_ip=$3
+worker_flags=$4
+
+hostname=$(hostname -f)
+mkdir -p /etc/rancher/rke2
+cat <<EOF >/etc/rancher/rke2/config.yaml
+server: https://${server_ip}:9345
+token:  "${token}"
+node-name: "${hostname}"
+EOF
+
+if [ -n "$worker_flags" ] && [[ "$worker_flags" == *":"* ]]
+then
+   echo "$worker_flags"
+   echo -e "$worker_flags" >> /etc/rancher/rke2/config.yaml
+   if [[ "$worker_flags" != *"cloud-provider-name"* ]]
+   then
+     echo -e "node-external-ip: $public_ip" >> /etc/rancher/rke2/config.yaml
+   fi
+   cat /etc/rancher/rke2/config.yaml
+else
+  echo -e "node-external-ip: $public_ip" >> /etc/rancher/rke2/config.yaml
+fi
+
+sudo systemctl restart rke2-agent

modules/install/join_rke2_master_eip.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+# This script is used to join one or more nodes as masters to the first master
+set -x
+echo "$@"
+
+initial_node_ip=$1
+token=$2
+public_ip=$3
+server_flags=$4
+
+hostname=$(hostname -f)
+mkdir -p /etc/rancher/rke2
+cat <<EOF >/etc/rancher/rke2/config.yaml
+write-kubeconfig-mode: "0644"
+tls-san:
+  - ${initial_node_ip}
+server: https://${initial_node_ip}:9345
+token:  "${token}"
+node-name: "${hostname}"
+EOF
+
+if [ -n "$server_flags" ] && [[ "$server_flags" == *":"* ]]
+then
+   echo "$server_flags"
+   echo -e "$server_flags" >> /etc/rancher/rke2/config.yaml
+   if [[ "$server_flags" != *"cloud-provider-name"* ]]
+   then
+     echo -e "node-external-ip: $public_ip" >> /etc/rancher/rke2/config.yaml
+   fi
+   cat /etc/rancher/rke2/config.yaml
+else
+  echo -e "node-external-ip: $public_ip" >> /etc/rancher/rke2/config.yaml
+fi
+
+sudo systemctl restart --no-block rke2-server

modules/install/rke2_stop_start_instace.sh

@@ -0,0 +1,41 @@
+#!/bin/bash  
+#set -x
+echo "$@"
+type=$1
+resource_name=$2
+
+cd /tmp
+if [ "$type" == "stop" ]; then    
+    #output instances IDs
+
+    aws ec2 describe-instances --filters "Name=tag:Name,Values=${resource_name}-server1,${resource_name}-server2" \
+     "Name=instance-state-name,Values=running" \
+    --output text --query 'Reservations[*].Instances[*].InstanceId' > /tmp/ids_server_1_2.txt
+
+    aws ec2 describe-instances --filters "Name=tag:Name,Values=${resource_name}-server,${resource_name}-worker" \
+     "Name=instance-state-name,Values=running" \
+    --output text --query 'Reservations[*].Instances[*].InstanceId' > /tmp/ids_master_worker.txt    
+
+    cat /tmp/ids_server_1_2.txt /tmp/ids_master_worker.txt > /tmp/ids_all.txt
+fi
+
+if [ "$type" == "stop" ]; then
+    file="ids_all.txt" 
+elif [ "$type" == "start_s1_s2" ]; then 
+    file="ids_server_1_2.txt" 
+elif [ "$type" == "start_master_worker" ]; then
+    file="ids_master_worker.txt" 
+fi
+
+i=1  
+while read line; do    
+    if [ "$type" == "stop" ]; then 
+        aws ec2 stop-instances --instance-ids $line 
+    elif  [[ "$type" == "start_s1_s2" || "$type" == "start_master_worker" ]]; then
+        aws ec2 start-instances --instance-ids $line
+        sleep 60
+    fi
+i=$((i+1))  
+done < $file 
+sleep 120
+

modules/rke2/main.tf

@@ -15,6 +15,7 @@ module "master" {
   cp_only_nodes      = var.cp_only_nodes
   cp_worker_nodes    = var.cp_worker_nodes
   optional_files     = var.optional_files
+  create_eip         = var.create_eip
 
   # AWS variables
   access_key         = var.access_key
@@ -55,6 +56,7 @@ module "worker" {
 
   # AWS variables
   access_key         = var.access_key
+  create_eip         = var.create_eip
   key_name           = var.key_name
   availability_zone  = var.availability_zone
   aws_ami            = var.aws_ami

modules/rke2/master/instances_server.tf

@@ -64,6 +64,97 @@ resource "aws_instance" "master" {
   }
 }
 
+
+locals {
+        eip_index = { for i, v in aws_instance.master2 : tonumber(i)  => v.id if var.create_eip}      
+}
+
+resource "aws_eip" "stop-eip-master" { 
+        count = var.create_eip ? 1 : 0
+        vpc = true
+        tags = {
+           Name ="${var.resource_name}-server"
+        }     
+}
+
+resource "aws_eip" "stop-eip-master2" {
+        for_each = local.eip_index
+        vpc = true
+          tags = {
+                  Name ="${var.resource_name}-servers-${each.key}"
+        }
+      depends_on = [aws_eip.stop-eip-master]
+}
+
+resource "aws_eip_association" "master-stop-association" { 
+        count = var.create_eip ? 1 : 0
+        instance_id = aws_instance.master.id
+        allocation_id = aws_eip.stop-eip-master[0].id
+        depends_on = [aws_eip.stop-eip-master]
+}
+
+resource "aws_eip_association" "master2-stop-association" {
+        for_each = local.eip_index
+        instance_id = aws_instance.master2[each.key].id
+        allocation_id = aws_eip.stop-eip-master2[each.key].id
+        depends_on = [aws_instance.master]
+}
+
+resource "null_resource" "master_eip" {
+  count = var.create_eip ? 1 : 0 
+  connection {
+    type        = "ssh"
+    user        = var.aws_user
+    host        = aws_eip.stop-eip-master[0].public_ip
+    private_key = file(var.access_key)
+  }
+
+  provisioner "file" {
+    source      = "../install/install_rke2_master_eip.sh"
+    destination = "/tmp/install_rke2_master_eip.sh"
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "chmod +x /tmp/install_rke2_master_eip.sh",
+      "sudo /tmp/install_rke2_master_eip.sh ${var.create_lb ? aws_route53_record.aws_route53[0].fqdn : "fake.fqdn.value"} ${aws_eip.stop-eip-master[0].public_ip}  \"${var.server_flags}\" ",
+    ]
+  }
+
+   provisioner "local-exec" {
+    command = "echo ${aws_eip.stop-eip-master[0].public_ip} > /tmp/${var.resource_name}_master_ip"
+  }
+
+   depends_on = [aws_instance.master, 
+                 aws_eip_association.master-stop-association]                 
+}
+
+resource "null_resource" "master2_eip" {
+  for_each = local.eip_index
+  connection {
+    type        = "ssh"
+    user        = var.aws_user
+    host        = tostring(aws_eip.stop-eip-master2[each.key].public_ip)
+    private_key = file(var.access_key)
+  }
+
+  provisioner "file" {
+    source      = "../install/join_rke2_master_eip.sh"
+    destination = "/tmp/join_rke2_master_eip.sh"
+  }
+
+  provisioner "remote-exec" { 
+    inline = [
+      "chmod +x /tmp/join_rke2_master_eip.sh",
+      "sudo /tmp/join_rke2_master_eip.sh  ${aws_eip.stop-eip-master[0].public_ip} ${local.node_token} ${aws_eip.stop-eip-master2[each.key].public_ip} \"${var.server_flags}\"",
+    ]
+  }
+
+ depends_on = [null_resource.master_eip, 
+                 aws_eip_association.master2-stop-association]
+}
+
+
 resource "aws_instance" "master2" {
   ami                  = var.aws_ami
   instance_type        = var.ec2_instance_class

modules/rke2/master/variables.tf

@@ -19,6 +19,7 @@ variable "install_mode" {}
 variable "install_method" {}
 variable "rke2_channel" {}
 variable "server_flags" {}
+variable "create_eip" {}
 variable "sg_id" {}
 variable "subnets" {}
 variable "vpc_id" {}

modules/rke2/variables.tf

@@ -19,6 +19,7 @@ variable "access_key" {}
 variable "key_name" {}
 variable "availability_zone" {}
 variable "aws_ami" {}
+variable "create_eip" {}
 variable "aws_user" {}
 variable "ec2_instance_class" {}
 variable "volume_size" {}

modules/rke2/worker/instances_worker.tf

@@ -38,6 +38,58 @@ resource "aws_instance" "worker" {
   }
 }
 
+locals {
+        eip_index = { for i, v in aws_instance.worker : tonumber(i)  => v.id if var.create_eip}      
+}
+
+resource "aws_eip" "worker-eip" {
+  depends_on = [var.dependency]
+  for_each = local.eip_index
+  vpc = true
+  tags = {
+      Name ="${var.resource_name}-worker-${each.key}"
+    }
+}
+
+resource "aws_eip_association" "worker-association" {
+        for_each = local.eip_index 
+        instance_id = aws_instance.worker[each.key].id
+        allocation_id = aws_eip.worker-eip[each.key].id
+        depends_on = [aws_instance.worker]
+} 
+
+resource "null_resource" "worker_eip" {
+
+  for_each = local.eip_index
+  connection {
+    type = "ssh"
+    user = var.aws_user
+    host = aws_eip.worker-eip[each.key].public_ip
+    private_key = "${file(var.access_key)}"
+  }
+
+  provisioner "file" {
+    source      = "../install/join_rke2_agent_eip.sh"
+    destination = "/tmp/join_rke2_agent_eip.sh"
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "chmod +x /tmp/join_rke2_agent_eip.sh",
+      "sudo /tmp/join_rke2_agent_eip.sh  ${local.master_ip} \"${local.node_token}\"  ${aws_eip.worker-eip[each.key].public_ip} \"${var.worker_flags}\" ",
+    ]
+     on_failure = "continue"
+  }
+
+provisioner "local-exec" {
+    command    = "sed s/127.0.0.1/\"${local.master_ip}\"/g /tmp/${var.resource_name}_config >/tmp/${var.resource_name}_kubeconfig"
+    on_failure = continue
+  }
+
+   depends_on = [aws_instance.worker, 
+                 aws_eip_association.worker-association]
+}
+
 data "local_file" "master_ip" {
   depends_on = [var.dependency]
   filename = "/tmp/${var.resource_name}_master_ip"
@@ -54,4 +106,37 @@ data "local_file" "token" {
 
 locals {
   node_token = trimspace("${data.local_file.token.content}")
+}
+
+resource "null_resource" "stop_resource" {
+  count = var.create_eip ? 1 : 0
+  depends_on = [null_resource.worker_eip]
+   provisioner "local-exec" {
+   command = "chmod +x ../install/rke2_stop_start_instace.sh"
+  }
+  provisioner "local-exec" {
+    command = "../install/rke2_stop_start_instace.sh stop ${var.resource_name}"
+  }
+}
+
+resource "time_sleep" "wait_for_stop" {
+  count = var.create_eip ? 1 : 0
+  create_duration = "400s"
+  depends_on = [null_resource.stop_resource]
+}
+
+resource "null_resource" "start_server1_server2" {
+  count = var.create_eip ? 1 : 0
+  depends_on = [time_sleep.wait_for_stop]
+  provisioner "local-exec" {
+    command = "../install/rke2_stop_start_instace.sh start_s1_s2 ${var.resource_name}"
+  }
+}
+
+resource "null_resource" "start_master_worker" {
+  count = var.create_eip ? 1 : 0
+  depends_on = [null_resource.start_server1_server2]
+  provisioner "local-exec" {
+    command = "../install/rke2_stop_start_instace.sh start_master_worker ${var.resource_name}"
+  }
 }

modules/rke2/worker/variables.tf

@@ -12,6 +12,7 @@ variable "volume_size" {}
 variable "iam_role" {}
 variable "node_os" {}
 variable "no_of_worker_nodes" {}
+variable "create_eip" {}
 variable "password" {
   default = "password"
 }