From 5194d8306e1e02c9236b59b868841c2375d502c3 Mon Sep 17 00:00:00 2001
From: Robin Wallace <robin.wallace@elastisys.com>
Date: Sat, 7 Oct 2023 04:45:41 +0200
Subject: [PATCH 1/8] upcloud: update terraform provider strict anti-affinity
 (#10474)

---
 contrib/terraform/upcloud/README.md           |   2 +-
 .../terraform/upcloud/cluster-settings.tfvars |   8 +-
 .../modules/kubernetes-cluster/main.tf        | 298 +++++++++---------
 .../modules/kubernetes-cluster/output.tf      |   8 +-
 .../modules/kubernetes-cluster/variables.tf   |  14 +-
 .../modules/kubernetes-cluster/versions.tf    |   4 +-
 .../upcloud/sample-inventory/cluster.tfvars   |  18 +-
 contrib/terraform/upcloud/variables.tf        |   4 +-
 contrib/terraform/upcloud/versions.tf         |   2 +-
 9 files changed, 179 insertions(+), 179 deletions(-)

diff --git a/contrib/terraform/upcloud/README.md b/contrib/terraform/upcloud/README.md
index c893c34acb4..6d35a42686e 100644
--- a/contrib/terraform/upcloud/README.md
+++ b/contrib/terraform/upcloud/README.md
@@ -140,4 +140,4 @@ terraform destroy --var-file cluster-settings.tfvars \
   * `backend_servers`: List of servers that traffic to the port should be forwarded to.
 * `server_groups`: Group servers together
   * `servers`: The servers that should be included in the group.
-  * `anti_affinity`: If anti-affinity should be enabled, try to spread the VMs out on separate nodes.
+  * `anti_affinity_policy`: Defines if a server group is an anti-affinity group. Setting this to "strict" or yes" will result in all servers in the group being placed on separate compute hosts. The value can be "strict", "yes" or "no". "strict" refers to strict policy doesn't allow servers in the same server group to be on the same host. "yes" refers to best-effort policy and tries to put servers on different hosts, but this is not guaranteed.
diff --git a/contrib/terraform/upcloud/cluster-settings.tfvars b/contrib/terraform/upcloud/cluster-settings.tfvars
index d88945f65e2..199661e6b1b 100644
--- a/contrib/terraform/upcloud/cluster-settings.tfvars
+++ b/contrib/terraform/upcloud/cluster-settings.tfvars
@@ -18,7 +18,7 @@ ssh_public_keys = [
 
 # check list of available plan https://developers.upcloud.com/1.3/7-plans/
 machines = {
-  "master-0" : {
+  "control-plane-0" : {
     "node_type" : "master",
     # plan to use instead of custom cpu/mem
     "plan" : null,
@@ -133,9 +133,9 @@ loadbalancers = {
 server_groups = {
   # "control-plane" = {
   #   servers = [
-  #     "master-0"
+  #     "control-plane-0"
   #   ]
-  #   anti_affinity = true
+  #   anti_affinity_policy = "strict"
   # },
   # "workers" = {
   #   servers = [
@@ -143,6 +143,6 @@ server_groups = {
   #     "worker-1",
   #     "worker-2"
   #   ]
-  #   anti_affinity = true
+  #   anti_affinity_policy = "yes"
   # }
 }
\ No newline at end of file
diff --git a/contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf b/contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf
index c2d43a3aced..72c44108740 100644
--- a/contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf
+++ b/contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf
@@ -3,7 +3,7 @@ locals {
   disks = flatten([
     for node_name, machine in var.machines : [
       for disk_name, disk in machine.additional_disks : {
-        disk = disk
+        disk      = disk
         disk_name = disk_name
         node_name = node_name
       }
@@ -13,8 +13,8 @@ locals {
   lb_backend_servers = flatten([
     for lb_name, loadbalancer in var.loadbalancers : [
       for backend_server in loadbalancer.backend_servers : {
-        port = loadbalancer.target_port
-        lb_name = lb_name
+        port        = loadbalancer.target_port
+        lb_name     = lb_name
         server_name = backend_server
       }
     ]
@@ -22,7 +22,7 @@ locals {
 
   # If prefix is set, all resources will be prefixed with "${var.prefix}-"
   # Else don't prefix with anything
-  resource-prefix = "%{ if var.prefix != ""}${var.prefix}-%{ endif }"
+  resource-prefix = "%{if var.prefix != ""}${var.prefix}-%{endif}"
 }
 
 resource "upcloud_network" "private" {
@@ -38,7 +38,7 @@ resource "upcloud_network" "private" {
 
 resource "upcloud_storage" "additional_disks" {
   for_each = {
-    for disk in local.disks: "${disk.node_name}_${disk.disk_name}" => disk.disk
+    for disk in local.disks : "${disk.node_name}_${disk.disk_name}" => disk.disk
   }
 
   size  = each.value.size
@@ -61,8 +61,8 @@ resource "upcloud_server" "master" {
   zone     = var.zone
 
   template {
-  storage = var.template_name
-  size    = each.value.disk_size
+    storage = var.template_name
+    size    = each.value.disk_size
   }
 
   # Public network interface
@@ -81,14 +81,14 @@ resource "upcloud_server" "master" {
     ignore_changes = [storage_devices]
   }
 
-  firewall  = var.firewall_enabled
+  firewall = var.firewall_enabled
 
   dynamic "storage_devices" {
     for_each = {
       for disk_key_name, disk in upcloud_storage.additional_disks :
-        disk_key_name => disk
-        # Only add the disk if it matches the node name in the start of its name
-        if length(regexall("^${each.key}_.+", disk_key_name)) > 0
+      disk_key_name => disk
+      # Only add the disk if it matches the node name in the start of its name
+      if length(regexall("^${each.key}_.+", disk_key_name)) > 0
     }
 
     content {
@@ -138,14 +138,14 @@ resource "upcloud_server" "worker" {
     ignore_changes = [storage_devices]
   }
 
-  firewall  = var.firewall_enabled
+  firewall = var.firewall_enabled
 
   dynamic "storage_devices" {
     for_each = {
       for disk_key_name, disk in upcloud_storage.additional_disks :
-        disk_key_name => disk
-        # Only add the disk if it matches the node name in the start of its name
-        if length(regexall("^${each.key}_.+", disk_key_name)) > 0
+      disk_key_name => disk
+      # Only add the disk if it matches the node name in the start of its name
+      if length(regexall("^${each.key}_.+", disk_key_name)) > 0
     }
 
     content {
@@ -162,10 +162,10 @@ resource "upcloud_server" "worker" {
 }
 
 resource "upcloud_firewall_rules" "master" {
-  for_each = upcloud_server.master
+  for_each  = upcloud_server.master
   server_id = each.value.id
 
-  dynamic firewall_rule {
+  dynamic "firewall_rule" {
     for_each = var.master_allowed_remote_ips
 
     content {
@@ -181,7 +181,7 @@ resource "upcloud_firewall_rules" "master" {
     }
   }
 
-  dynamic firewall_rule {
+  dynamic "firewall_rule" {
     for_each = length(var.master_allowed_remote_ips) > 0 ? [1] : []
 
     content {
@@ -197,7 +197,7 @@ resource "upcloud_firewall_rules" "master" {
     }
   }
 
-  dynamic firewall_rule {
+  dynamic "firewall_rule" {
     for_each = var.k8s_allowed_remote_ips
 
     content {
@@ -213,7 +213,7 @@ resource "upcloud_firewall_rules" "master" {
     }
   }
 
-  dynamic firewall_rule {
+  dynamic "firewall_rule" {
     for_each = length(var.k8s_allowed_remote_ips) > 0 ? [1] : []
 
     content {
@@ -229,7 +229,7 @@ resource "upcloud_firewall_rules" "master" {
     }
   }
 
-  dynamic firewall_rule {
+  dynamic "firewall_rule" {
     for_each = var.master_allowed_ports
 
     content {
@@ -245,97 +245,97 @@ resource "upcloud_firewall_rules" "master" {
     }
   }
 
-  dynamic firewall_rule {
+  dynamic "firewall_rule" {
     for_each = var.firewall_default_deny_in ? ["tcp", "udp"] : []
 
     content {
-      action                 = "accept"
-      comment                = "UpCloud DNS"
-      source_port_end        = "53"
-      source_port_start      = "53"
-      direction              = "in"
-      family                 = "IPv4"
-      protocol               = firewall_rule.value
-      source_address_end     = "94.237.40.9"
-      source_address_start   = "94.237.40.9"
+      action               = "accept"
+      comment              = "UpCloud DNS"
+      source_port_end      = "53"
+      source_port_start    = "53"
+      direction            = "in"
+      family               = "IPv4"
+      protocol             = firewall_rule.value
+      source_address_end   = "94.237.40.9"
+      source_address_start = "94.237.40.9"
     }
   }
 
-  dynamic firewall_rule {
+  dynamic "firewall_rule" {
     for_each = var.firewall_default_deny_in ? ["tcp", "udp"] : []
 
     content {
-      action                 = "accept"
-      comment                = "UpCloud DNS"
-      source_port_end        = "53"
-      source_port_start      = "53"
-      direction              = "in"
-      family                 = "IPv4"
-      protocol               = firewall_rule.value
-      source_address_end     = "94.237.127.9"
-      source_address_start   = "94.237.127.9"
+      action               = "accept"
+      comment              = "UpCloud DNS"
+      source_port_end      = "53"
+      source_port_start    = "53"
+      direction            = "in"
+      family               = "IPv4"
+      protocol             = firewall_rule.value
+      source_address_end   = "94.237.127.9"
+      source_address_start = "94.237.127.9"
     }
   }
 
-  dynamic firewall_rule {
+  dynamic "firewall_rule" {
     for_each = var.firewall_default_deny_in ? ["tcp", "udp"] : []
 
     content {
-      action                 = "accept"
-      comment                = "UpCloud DNS"
-      source_port_end        = "53"
-      source_port_start      = "53"
-      direction              = "in"
-      family                 = "IPv6"
-      protocol               = firewall_rule.value
-      source_address_end     = "2a04:3540:53::1"
-      source_address_start   = "2a04:3540:53::1"
+      action               = "accept"
+      comment              = "UpCloud DNS"
+      source_port_end      = "53"
+      source_port_start    = "53"
+      direction            = "in"
+      family               = "IPv6"
+      protocol             = firewall_rule.value
+      source_address_end   = "2a04:3540:53::1"
+      source_address_start = "2a04:3540:53::1"
     }
   }
 
-  dynamic firewall_rule {
+  dynamic "firewall_rule" {
     for_each = var.firewall_default_deny_in ? ["tcp", "udp"] : []
 
     content {
-      action                 = "accept"
-      comment                = "UpCloud DNS"
-      source_port_end        = "53"
-      source_port_start      = "53"
-      direction              = "in"
-      family                 = "IPv6"
-      protocol               = firewall_rule.value
-      source_address_end     = "2a04:3544:53::1"
-      source_address_start   = "2a04:3544:53::1"
+      action               = "accept"
+      comment              = "UpCloud DNS"
+      source_port_end      = "53"
+      source_port_start    = "53"
+      direction            = "in"
+      family               = "IPv6"
+      protocol             = firewall_rule.value
+      source_address_end   = "2a04:3544:53::1"
+      source_address_start = "2a04:3544:53::1"
     }
   }
 
-  dynamic firewall_rule {
+  dynamic "firewall_rule" {
     for_each = var.firewall_default_deny_in ? ["udp"] : []
 
     content {
-      action                 = "accept"
-      comment                = "NTP Port"
-      source_port_end        = "123"
-      source_port_start      = "123"
-      direction              = "in"
-      family                 = "IPv4"
-      protocol               = firewall_rule.value
-      source_address_end     = "255.255.255.255"
-      source_address_start   = "0.0.0.0"
+      action               = "accept"
+      comment              = "NTP Port"
+      source_port_end      = "123"
+      source_port_start    = "123"
+      direction            = "in"
+      family               = "IPv4"
+      protocol             = firewall_rule.value
+      source_address_end   = "255.255.255.255"
+      source_address_start = "0.0.0.0"
     }
   }
 
-  dynamic firewall_rule {
+  dynamic "firewall_rule" {
     for_each = var.firewall_default_deny_in ? ["udp"] : []
 
     content {
-      action                 = "accept"
-      comment                = "NTP Port"
-      source_port_end        = "123"
-      source_port_start      = "123"
-      direction              = "in"
-      family                 = "IPv6"
-      protocol               = firewall_rule.value
+      action            = "accept"
+      comment           = "NTP Port"
+      source_port_end   = "123"
+      source_port_start = "123"
+      direction         = "in"
+      family            = "IPv6"
+      protocol          = firewall_rule.value
     }
   }
 
@@ -351,10 +351,10 @@ resource "upcloud_firewall_rules" "master" {
 }
 
 resource "upcloud_firewall_rules" "k8s" {
-  for_each = upcloud_server.worker
+  for_each  = upcloud_server.worker
   server_id = each.value.id
 
-  dynamic firewall_rule {
+  dynamic "firewall_rule" {
     for_each = var.k8s_allowed_remote_ips
 
     content {
@@ -370,7 +370,7 @@ resource "upcloud_firewall_rules" "k8s" {
     }
   }
 
-  dynamic firewall_rule {
+  dynamic "firewall_rule" {
     for_each = length(var.k8s_allowed_remote_ips) > 0 ? [1] : []
 
     content {
@@ -386,7 +386,7 @@ resource "upcloud_firewall_rules" "k8s" {
     }
   }
 
-  dynamic firewall_rule {
+  dynamic "firewall_rule" {
     for_each = var.worker_allowed_ports
 
     content {
@@ -402,97 +402,97 @@ resource "upcloud_firewall_rules" "k8s" {
     }
   }
 
-  dynamic firewall_rule {
+  dynamic "firewall_rule" {
     for_each = var.firewall_default_deny_in ? ["tcp", "udp"] : []
 
     content {
-      action                 = "accept"
-      comment                = "UpCloud DNS"
-      source_port_end        = "53"
-      source_port_start      = "53"
-      direction              = "in"
-      family                 = "IPv4"
-      protocol               = firewall_rule.value
-      source_address_end     = "94.237.40.9"
-      source_address_start   = "94.237.40.9"
+      action               = "accept"
+      comment              = "UpCloud DNS"
+      source_port_end      = "53"
+      source_port_start    = "53"
+      direction            = "in"
+      family               = "IPv4"
+      protocol             = firewall_rule.value
+      source_address_end   = "94.237.40.9"
+      source_address_start = "94.237.40.9"
     }
   }
 
-  dynamic firewall_rule {
+  dynamic "firewall_rule" {
     for_each = var.firewall_default_deny_in ? ["tcp", "udp"] : []
 
     content {
-      action                 = "accept"
-      comment                = "UpCloud DNS"
-      source_port_end        = "53"
-      source_port_start      = "53"
-      direction              = "in"
-      family                 = "IPv4"
-      protocol               = firewall_rule.value
-      source_address_end     = "94.237.127.9"
-      source_address_start   = "94.237.127.9"
+      action               = "accept"
+      comment              = "UpCloud DNS"
+      source_port_end      = "53"
+      source_port_start    = "53"
+      direction            = "in"
+      family               = "IPv4"
+      protocol             = firewall_rule.value
+      source_address_end   = "94.237.127.9"
+      source_address_start = "94.237.127.9"
     }
   }
 
-  dynamic firewall_rule {
+  dynamic "firewall_rule" {
     for_each = var.firewall_default_deny_in ? ["tcp", "udp"] : []
 
     content {
-      action                 = "accept"
-      comment                = "UpCloud DNS"
-      source_port_end        = "53"
-      source_port_start      = "53"
-      direction              = "in"
-      family                 = "IPv6"
-      protocol               = firewall_rule.value
-      source_address_end     = "2a04:3540:53::1"
-      source_address_start   = "2a04:3540:53::1"
+      action               = "accept"
+      comment              = "UpCloud DNS"
+      source_port_end      = "53"
+      source_port_start    = "53"
+      direction            = "in"
+      family               = "IPv6"
+      protocol             = firewall_rule.value
+      source_address_end   = "2a04:3540:53::1"
+      source_address_start = "2a04:3540:53::1"
     }
   }
 
-  dynamic firewall_rule {
+  dynamic "firewall_rule" {
     for_each = var.firewall_default_deny_in ? ["tcp", "udp"] : []
 
     content {
-      action                 = "accept"
-      comment                = "UpCloud DNS"
-      source_port_end        = "53"
-      source_port_start      = "53"
-      direction              = "in"
-      family                 = "IPv6"
-      protocol               = firewall_rule.value
-      source_address_end     = "2a04:3544:53::1"
-      source_address_start   = "2a04:3544:53::1"
+      action               = "accept"
+      comment              = "UpCloud DNS"
+      source_port_end      = "53"
+      source_port_start    = "53"
+      direction            = "in"
+      family               = "IPv6"
+      protocol             = firewall_rule.value
+      source_address_end   = "2a04:3544:53::1"
+      source_address_start = "2a04:3544:53::1"
     }
   }
 
-  dynamic firewall_rule {
+  dynamic "firewall_rule" {
     for_each = var.firewall_default_deny_in ? ["udp"] : []
 
     content {
-      action                 = "accept"
-      comment                = "NTP Port"
-      source_port_end        = "123"
-      source_port_start      = "123"
-      direction              = "in"
-      family                 = "IPv4"
-      protocol               = firewall_rule.value
-      source_address_end     = "255.255.255.255"
-      source_address_start   = "0.0.0.0"
+      action               = "accept"
+      comment              = "NTP Port"
+      source_port_end      = "123"
+      source_port_start    = "123"
+      direction            = "in"
+      family               = "IPv4"
+      protocol             = firewall_rule.value
+      source_address_end   = "255.255.255.255"
+      source_address_start = "0.0.0.0"
     }
   }
 
-  dynamic firewall_rule {
+  dynamic "firewall_rule" {
     for_each = var.firewall_default_deny_in ? ["udp"] : []
 
     content {
-      action                 = "accept"
-      comment                = "NTP Port"
-      source_port_end        = "123"
-      source_port_start      = "123"
-      direction              = "in"
-      family                 = "IPv6"
-      protocol               = firewall_rule.value
+      action            = "accept"
+      comment           = "NTP Port"
+      source_port_end   = "123"
+      source_port_start = "123"
+      direction         = "in"
+      family            = "IPv6"
+      protocol          = firewall_rule.value
     }
   }
 
@@ -535,9 +535,9 @@ resource "upcloud_loadbalancer_frontend" "lb_frontend" {
 
 resource "upcloud_loadbalancer_static_backend_member" "lb_backend_member" {
   for_each = {
-    for be_server in local.lb_backend_servers:
-      "${be_server.server_name}-lb-backend-${be_server.lb_name}" => be_server
-      if var.loadbalancer_enabled
+    for be_server in local.lb_backend_servers :
+    "${be_server.server_name}-lb-backend-${be_server.lb_name}" => be_server
+    if var.loadbalancer_enabled
   }
 
   backend      = upcloud_loadbalancer_backend.lb_backend[each.value.lb_name].id
@@ -550,9 +550,9 @@ resource "upcloud_loadbalancer_static_backend_member" "lb_backend_member" {
 }
 
 resource "upcloud_server_group" "server_groups" {
-  for_each      = var.server_groups
-  title         = each.key
-  anti_affinity = each.value.anti_affinity
-  labels        = {}
-  members       = [for server in each.value.servers : merge(upcloud_server.master, upcloud_server.worker)[server].id]
+  for_each             = var.server_groups
+  title                = each.key
+  anti_affinity_policy = each.value.anti_affinity_policy
+  labels               = {}
+  members              = [for server in each.value.servers : merge(upcloud_server.master, upcloud_server.worker)[server].id]
 }
\ No newline at end of file
diff --git a/contrib/terraform/upcloud/modules/kubernetes-cluster/output.tf b/contrib/terraform/upcloud/modules/kubernetes-cluster/output.tf
index c1f8c7c9c79..084f02348e6 100644
--- a/contrib/terraform/upcloud/modules/kubernetes-cluster/output.tf
+++ b/contrib/terraform/upcloud/modules/kubernetes-cluster/output.tf
@@ -3,8 +3,8 @@ output "master_ip" {
   value = {
     for instance in upcloud_server.master :
     instance.hostname => {
-      "public_ip": instance.network_interface[0].ip_address
-      "private_ip": instance.network_interface[1].ip_address
+      "public_ip" : instance.network_interface[0].ip_address
+      "private_ip" : instance.network_interface[1].ip_address
     }
   }
 }
@@ -13,8 +13,8 @@ output "worker_ip" {
   value = {
     for instance in upcloud_server.worker :
     instance.hostname => {
-      "public_ip": instance.network_interface[0].ip_address
-      "private_ip": instance.network_interface[1].ip_address
+      "public_ip" : instance.network_interface[0].ip_address
+      "private_ip" : instance.network_interface[1].ip_address
     }
   }
 }
diff --git a/contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf b/contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf
index 8c492ae2a30..a99d2d8488d 100644
--- a/contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf
+++ b/contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf
@@ -15,11 +15,11 @@ variable "private_network_cidr" {}
 variable "machines" {
   description = "Cluster machines"
   type = map(object({
-    node_type       = string
-    plan            = string
-    cpu             = string
-    mem             = string
-    disk_size       =  number
+    node_type = string
+    plan      = string
+    cpu       = string
+    mem       = string
+    disk_size = number
     additional_disks = map(object({
       size = number
       tier = string
@@ -99,7 +99,7 @@ variable "server_groups" {
   description = "Server groups"
 
   type = map(object({
-    anti_affinity   = bool
-    servers = list(string)
+    anti_affinity_policy = string
+    servers              = list(string)
   }))
 }
\ No newline at end of file
diff --git a/contrib/terraform/upcloud/modules/kubernetes-cluster/versions.tf b/contrib/terraform/upcloud/modules/kubernetes-cluster/versions.tf
index 75230b94907..3138453bef5 100644
--- a/contrib/terraform/upcloud/modules/kubernetes-cluster/versions.tf
+++ b/contrib/terraform/upcloud/modules/kubernetes-cluster/versions.tf
@@ -2,8 +2,8 @@
 terraform {
   required_providers {
     upcloud = {
-      source = "UpCloudLtd/upcloud"
-      version = "~>2.7.1"
+      source  = "UpCloudLtd/upcloud"
+      version = "~>2.12.0"
     }
   }
   required_version = ">= 0.13"
diff --git a/contrib/terraform/upcloud/sample-inventory/cluster.tfvars b/contrib/terraform/upcloud/sample-inventory/cluster.tfvars
index 4e8ade831a9..58536674f68 100644
--- a/contrib/terraform/upcloud/sample-inventory/cluster.tfvars
+++ b/contrib/terraform/upcloud/sample-inventory/cluster.tfvars
@@ -18,7 +18,7 @@ ssh_public_keys = [
 
 # check list of available plan https://developers.upcloud.com/1.3/7-plans/
 machines = {
-  "master-0" : {
+  "control-plane-0" : {
     "node_type" : "master",
     # plan to use instead of custom cpu/mem
     "plan" : null,
@@ -28,7 +28,7 @@ machines = {
     "mem" : "4096"
     # The size of the storage in GB
     "disk_size" : 250
-    "additional_disks": {}
+    "additional_disks" : {}
   },
   "worker-0" : {
     "node_type" : "worker",
@@ -40,7 +40,7 @@ machines = {
     "mem" : "4096"
     # The size of the storage in GB
     "disk_size" : 250
-    "additional_disks": {
+    "additional_disks" : {
       # "some-disk-name-1": {
       #   "size": 100,
       #   "tier": "maxiops",
@@ -61,7 +61,7 @@ machines = {
     "mem" : "4096"
     # The size of the storage in GB
     "disk_size" : 250
-    "additional_disks": {
+    "additional_disks" : {
       # "some-disk-name-1": {
       #   "size": 100,
       #   "tier": "maxiops",
@@ -82,7 +82,7 @@ machines = {
     "mem" : "4096"
     # The size of the storage in GB
     "disk_size" : 250
-    "additional_disks": {
+    "additional_disks" : {
       # "some-disk-name-1": {
       #   "size": 100,
       #   "tier": "maxiops",
@@ -118,7 +118,7 @@ master_allowed_ports = []
 worker_allowed_ports = []
 
 loadbalancer_enabled = false
-loadbalancer_plan = "development"
+loadbalancer_plan    = "development"
 loadbalancers = {
   # "http" : {
   #   "port" : 80,
@@ -134,9 +134,9 @@ loadbalancers = {
 server_groups = {
   # "control-plane" = {
   #   servers = [
-  #     "master-0"
+  #     "control-plane-0"
   #   ]
-  #   anti_affinity = true
+  #   anti_affinity_policy = "strict"
   # },
   # "workers" = {
   #   servers = [
@@ -144,6 +144,6 @@ server_groups = {
   #     "worker-1",
   #     "worker-2"
   #   ]
-  #   anti_affinity = true
+  #   anti_affinity_policy = "yes"
   # }
 }
\ No newline at end of file
diff --git a/contrib/terraform/upcloud/variables.tf b/contrib/terraform/upcloud/variables.tf
index 3b2c55804a6..880dc415779 100644
--- a/contrib/terraform/upcloud/variables.tf
+++ b/contrib/terraform/upcloud/variables.tf
@@ -136,8 +136,8 @@ variable "server_groups" {
   description = "Server groups"
 
   type = map(object({
-    anti_affinity = bool
-    servers       = list(string)
+    anti_affinity_policy = string
+    servers              = list(string)
   }))
 
   default = {}
diff --git a/contrib/terraform/upcloud/versions.tf b/contrib/terraform/upcloud/versions.tf
index 99507471cba..3138453bef5 100644
--- a/contrib/terraform/upcloud/versions.tf
+++ b/contrib/terraform/upcloud/versions.tf
@@ -3,7 +3,7 @@ terraform {
   required_providers {
     upcloud = {
       source  = "UpCloudLtd/upcloud"
-      version = "~>2.7.1"
+      version = "~>2.12.0"
     }
   }
   required_version = ">= 0.13"

From bea5034ddf543244bea64c2a341828d60e6cf50b Mon Sep 17 00:00:00 2001
From: Mateus Caruccio <mateus.caruccio@getupcloud.com>
Date: Fri, 6 Oct 2023 23:52:35 -0300
Subject: [PATCH 2/8] Update metallb example configs (#10485)

---
 .../sample/group_vars/k8s_cluster/addons.yml  | 46 ++++++++++---------
 1 file changed, 24 insertions(+), 22 deletions(-)

diff --git a/inventory/sample/group_vars/k8s_cluster/addons.yml b/inventory/sample/group_vars/k8s_cluster/addons.yml
index 38bc8caa97e..8882b5212fd 100644
--- a/inventory/sample/group_vars/k8s_cluster/addons.yml
+++ b/inventory/sample/group_vars/k8s_cluster/addons.yml
@@ -176,33 +176,35 @@ cert_manager_enabled: false
 # MetalLB deployment
 metallb_enabled: false
 metallb_speaker_enabled: "{{ metallb_enabled }}"
-# metallb_speaker_nodeselector:
-#   kubernetes.io/os: "linux"
-# metallb_controller_nodeselector:
-#   kubernetes.io/os: "linux"
-# metallb_speaker_tolerations:
-#   - key: "node-role.kubernetes.io/master"
-#     operator: "Equal"
-#     value: ""
-#     effect: "NoSchedule"
-#   - key: "node-role.kubernetes.io/control-plane"
-#     operator: "Equal"
-#     value: ""
-#     effect: "NoSchedule"
-# metallb_controller_tolerations:
-#   - key: "node-role.kubernetes.io/master"
-#     operator: "Equal"
-#     value: ""
-#     effect: "NoSchedule"
-#   - key: "node-role.kubernetes.io/control-plane"
-#     operator: "Equal"
-#     value: ""
-#     effect: "NoSchedule"
 # metallb_version: v0.13.9
 # metallb_protocol: "layer2"
 # metallb_port: "7472"
 # metallb_memberlist_port: "7946"
 # metallb_config:
+#   speaker:
+#     nodeselector:
+#       kubernetes.io/os: "linux"
+#     tollerations:
+#       - key: "node-role.kubernetes.io/master"
+#         operator: "Equal"
+#         value: ""
+#         effect: "NoSchedule"
+#       - key: "node-role.kubernetes.io/control-plane"
+#         operator: "Equal"
+#         value: ""
+#         effect: "NoSchedule"
+#   controller
+#     nodeselector:
+#       kubernetes.io/os: "linux"
+#     tolerations:
+#       - key: "node-role.kubernetes.io/master"
+#         operator: "Equal"
+#         value: ""
+#         effect: "NoSchedule"
+#       - key: "node-role.kubernetes.io/control-plane"
+#         operator: "Equal"
+#         value: ""
+#         effect: "NoSchedule"
 #   address_pools:
 #     primary:
 #       ip_range:

From acb86c23f944bb3107a18ba6324ac14db9ba7ae4 Mon Sep 17 00:00:00 2001
From: Ross Kusler <rosskus@gmail.com>
Date: Fri, 6 Oct 2023 19:52:45 -0700
Subject: [PATCH 3/8] [kube-router] Add option to disable bgp-graceful-restart
 (10488) (#10489)

---
 .../sample/group_vars/k8s_cluster/k8s-net-kube-router.yml      | 3 +++
 roles/network_plugin/kube-router/defaults/main.yml             | 3 +++
 roles/network_plugin/kube-router/templates/kube-router.yml.j2  | 2 +-
 3 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/inventory/sample/group_vars/k8s_cluster/k8s-net-kube-router.yml b/inventory/sample/group_vars/k8s_cluster/k8s-net-kube-router.yml
index e4dfcc9b58d..144b381029a 100644
--- a/inventory/sample/group_vars/k8s_cluster/k8s-net-kube-router.yml
+++ b/inventory/sample/group_vars/k8s_cluster/k8s-net-kube-router.yml
@@ -19,6 +19,9 @@
 # Add LoadBalancer IP of service status as set by the LB provider to the RIB so that it gets advertised to the BGP peers.
 # kube_router_advertise_loadbalancer_ip: false
 
+# Enables BGP graceful restarts
+# kube_router_bgp_graceful_restart: true
+
 # Adjust manifest of kube-router daemonset template with DSR needed changes
 # kube_router_enable_dsr: false
 
diff --git a/roles/network_plugin/kube-router/defaults/main.yml b/roles/network_plugin/kube-router/defaults/main.yml
index 5d4dccc34f1..c01a3532bd8 100644
--- a/roles/network_plugin/kube-router/defaults/main.yml
+++ b/roles/network_plugin/kube-router/defaults/main.yml
@@ -18,6 +18,9 @@ kube_router_advertise_external_ip: false
 # Add LoadBalancer IP of service status as set by the LB provider to the RIB so that it gets advertised to the BGP peers.
 kube_router_advertise_loadbalancer_ip: false
 
+# Enables BGP graceful restarts
+kube_router_bgp_graceful_restart: true
+
 # Adjust manifest of kube-router daemonset template with DSR needed changes
 kube_router_enable_dsr: false
 
diff --git a/roles/network_plugin/kube-router/templates/kube-router.yml.j2 b/roles/network_plugin/kube-router/templates/kube-router.yml.j2
index ab677abc889..89ded6e4927 100644
--- a/roles/network_plugin/kube-router/templates/kube-router.yml.j2
+++ b/roles/network_plugin/kube-router/templates/kube-router.yml.j2
@@ -39,7 +39,7 @@ spec:
         - --run-firewall={{ kube_router_run_firewall | bool }}
         - --run-service-proxy={{ kube_router_run_service_proxy | bool }}
         - --kubeconfig=/var/lib/kube-router/kubeconfig
-        - --bgp-graceful-restart=true
+        - --bgp-graceful-restart={{ kube_router_bgp_graceful_restart }}
 {% if kube_router_advertise_cluster_ip %}
         - --advertise-cluster-ip
 {% endif %}

From abaddb4c9b6fc72a92c630584776cc07678f4eb6 Mon Sep 17 00:00:00 2001
From: reza <rezagh1377@gmail.com>
Date: Mon, 9 Oct 2023 10:48:16 +0330
Subject: [PATCH 4/8] fixed the bug in
 kubespray/contrib/offline/manage-offline-files.sh

---
 contrib/offline/manage-offline-files.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/offline/manage-offline-files.sh b/contrib/offline/manage-offline-files.sh
index 41936cdef08..875bd5609f9 100755
--- a/contrib/offline/manage-offline-files.sh
+++ b/contrib/offline/manage-offline-files.sh
@@ -38,7 +38,7 @@ sudo "${runtime}" container inspect nginx >/dev/null 2>&1
 if [ $? -ne 0 ]; then
     sudo "${runtime}" run \
         --restart=always -d -p ${NGINX_PORT}:80 \
-        --volume "${OFFLINE_FILES_DIR}:/usr/share/nginx/html/download" \
+        --volume "${OFFLINE_FILES_DIR}":/usr/share/nginx/html/download \
         --volume "${CURRENT_DIR}"/nginx.conf:/etc/nginx/nginx.conf \
         --name nginx nginx:alpine
 fi

From d3101d65aa5da124aafc610fbb0d42feedb33326 Mon Sep 17 00:00:00 2001
From: Elias-elastisys <112404905+Elias-elastisys@users.noreply.github.com>
Date: Tue, 10 Oct 2023 14:32:41 +0200
Subject: [PATCH 5/8] Added templating to coredns error to allow for
 consolidation (#10501)

---
 roles/kubernetes-apps/ansible/defaults/main.yml  |  2 ++
 .../ansible/templates/coredns-config.yml.j2      | 16 ++++++++++++----
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/roles/kubernetes-apps/ansible/defaults/main.yml b/roles/kubernetes-apps/ansible/defaults/main.yml
index 4de1fe916f6..4e8b23356bd 100644
--- a/roles/kubernetes-apps/ansible/defaults/main.yml
+++ b/roles/kubernetes-apps/ansible/defaults/main.yml
@@ -27,6 +27,8 @@ coredns_default_zone_cache_block: |
 #     answer name (.*)\.svc\.cluster\.local {1}.my.domain
 #   }
 
+# coredns_additional_error_config: |
+#   consolidate 5m ".* i/o timeout$" warning
 
 # dns_upstream_forward_extra_opts apply to coredns forward section as well as nodelocaldns upstream target forward section
 # dns_upstream_forward_extra_opts:
diff --git a/roles/kubernetes-apps/ansible/templates/coredns-config.yml.j2 b/roles/kubernetes-apps/ansible/templates/coredns-config.yml.j2
index 7a06023e84b..004ce0643d1 100644
--- a/roles/kubernetes-apps/ansible/templates/coredns-config.yml.j2
+++ b/roles/kubernetes-apps/ansible/templates/coredns-config.yml.j2
@@ -12,7 +12,11 @@ data:
 {%   for block in coredns_external_zones %}
     {{ block['zones'] | join(' ') }} {
         log
-        errors
+        errors {
+{% if coredns_additional_error_config is defined %}
+          {{ coredns_additional_error_config | indent(width=10, first=False) }}
+{% endif %}
+        }
 {% if block['rewrite'] is defined and block['rewrite'] | length > 0 %}
 {% for rewrite_match in block['rewrite'] %}
         rewrite {{ rewrite_match }}
@@ -31,10 +35,14 @@ data:
 {%   endfor %}
 {% endif %}
     .:53 {
-        {% if coredns_additional_configs is defined %}
+{% if coredns_additional_configs is defined %}
         {{ coredns_additional_configs | indent(width=8, first=False) }}
-        {% endif %}
-        errors
+{% endif %}
+        errors {
+{% if coredns_additional_error_config is defined %}
+          {{ coredns_additional_error_config | indent(width=10, first=False) }}
+{% endif %}
+        }
         health {
             lameduck 5s
         }

From b2d8ec68a4dbbe1c2217679d5036dd120cd15c0f Mon Sep 17 00:00:00 2001
From: ERIK <bo.jiang@daocloud.io>
Date: Wed, 11 Oct 2023 11:50:37 +0800
Subject: [PATCH 6/8] Fix restart network task cannot be skipped (#10512)

Signed-off-by: bo.jiang <bo.jiang@daocloud.io>
---
 roles/reset/tasks/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/reset/tasks/main.yml b/roles/reset/tasks/main.yml
index 198b2c4903e..53fb9f44bb6 100644
--- a/roles/reset/tasks/main.yml
+++ b/roles/reset/tasks/main.yml
@@ -433,7 +433,7 @@
     state: restarted
   when:
     - ansible_os_family not in ["Flatcar", "Flatcar Container Linux by Kinvolk"]
-    - reset_restart_network
+    - reset_restart_network | bool
   tags:
     - services
     - network

From 4a8a47d438f57614e7224e34b0826892b7e5c12d Mon Sep 17 00:00:00 2001
From: Mohamed Omar Zaian <mohamedzaian@gmail.com>
Date: Wed, 11 Oct 2023 23:49:16 +0200
Subject: [PATCH 7/8] [ingress-nginx] upgrade to 1.9.0 (#10493)

---
 README.md                             | 2 +-
 roles/download/defaults/main/main.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 7caa9f34c36..883e342d77f 100644
--- a/README.md
+++ b/README.md
@@ -179,7 +179,7 @@ Note: Upstart/SysV init based OS types are not supported.
 - Application
   - [cert-manager](https://github.com/jetstack/cert-manager) v1.11.1
   - [coredns](https://github.com/coredns/coredns) v1.10.1
-  - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v1.8.2
+  - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v1.9.0
   - [krew](https://github.com/kubernetes-sigs/krew) v0.4.4
   - [argocd](https://argoproj.github.io/) v2.8.0
   - [helm](https://helm.sh/) v3.12.3
diff --git a/roles/download/defaults/main/main.yml b/roles/download/defaults/main/main.yml
index 3c3369460cd..0637746fbd6 100644
--- a/roles/download/defaults/main/main.yml
+++ b/roles/download/defaults/main/main.yml
@@ -306,7 +306,7 @@ rbd_provisioner_image_tag: "{{ rbd_provisioner_version }}"
 local_path_provisioner_version: "v0.0.24"
 local_path_provisioner_image_repo: "{{ docker_image_repo }}/rancher/local-path-provisioner"
 local_path_provisioner_image_tag: "{{ local_path_provisioner_version }}"
-ingress_nginx_version: "v1.8.2"
+ingress_nginx_version: "v1.9.0"
 ingress_nginx_controller_image_repo: "{{ kube_image_repo }}/ingress-nginx/controller"
 ingress_nginx_controller_image_tag: "{{ ingress_nginx_version }}"
 ingress_nginx_kube_webhook_certgen_image_repo: "{{ kube_image_repo }}/ingress-nginx/kube-webhook-certgen"

From e65050d3f4dcbb5c46034b19e69f459b104a3a40 Mon Sep 17 00:00:00 2001
From: emiran-orange <71817149+emiran-orange@users.noreply.github.com>
Date: Fri, 13 Oct 2023 04:06:04 +0200
Subject: [PATCH 8/8] Ability to define GPG key path for Docker APT (#10513)

---
 docs/docker.md                                  | 6 ++++++
 roles/container-engine/docker/defaults/main.yml | 3 +++
 roles/container-engine/docker/tasks/main.yml    | 1 +
 tests/files/packet_debian12-docker.yml          | 1 +
 tests/files/packet_ubuntu22-aio-docker.yml      | 1 +
 5 files changed, 12 insertions(+)

diff --git a/docs/docker.md b/docs/docker.md
index 4cfcb7fe260..b5de70274c4 100644
--- a/docs/docker.md
+++ b/docs/docker.md
@@ -97,3 +97,9 @@ Adding extra options to pass to the docker daemon:
 ## This string should be exactly as you wish it to appear.
 docker_options: ""
 ```
+
+For Debian based distributions, set the path to store the GPG key to avoid using the default one used in `apt_key` module (e.g. /etc/apt/trusted.gpg)
+
+```yaml
+docker_repo_key_keyring: /etc/apt/trusted.gpg.d/docker.gpg
+```
diff --git a/roles/container-engine/docker/defaults/main.yml b/roles/container-engine/docker/defaults/main.yml
index 91227f91e79..e537558c34b 100644
--- a/roles/container-engine/docker/defaults/main.yml
+++ b/roles/container-engine/docker/defaults/main.yml
@@ -5,6 +5,9 @@ docker_cli_version: "{{ docker_version }}"
 docker_package_info:
   pkgs:
 
+# Path where to store repo key
+# docker_repo_key_keyring: /etc/apt/trusted.gpg.d/docker.gpg
+
 docker_repo_key_info:
   repo_keys:
 
diff --git a/roles/container-engine/docker/tasks/main.yml b/roles/container-engine/docker/tasks/main.yml
index cf81ce2b1a6..1fc490b12e5 100644
--- a/roles/container-engine/docker/tasks/main.yml
+++ b/roles/container-engine/docker/tasks/main.yml
@@ -57,6 +57,7 @@
   apt_key:
     id: "{{ item }}"
     url: "{{ docker_repo_key_info.url }}"
+    keyring: "{{ docker_repo_key_keyring|default(omit) }}"
     state: present
   register: keyserver_task_result
   until: keyserver_task_result is succeeded
diff --git a/tests/files/packet_debian12-docker.yml b/tests/files/packet_debian12-docker.yml
index 5d4ac539f24..2a740610347 100644
--- a/tests/files/packet_debian12-docker.yml
+++ b/tests/files/packet_debian12-docker.yml
@@ -7,3 +7,4 @@ mode: default
 container_manager: docker
 etcd_deployment_type: docker
 resolvconf_mode: docker_dns
+docker_repo_key_keyring: /etc/apt/trusted.gpg.d/docker.gpg
diff --git a/tests/files/packet_ubuntu22-aio-docker.yml b/tests/files/packet_ubuntu22-aio-docker.yml
index b78c6b0a473..d0f9e70dca7 100644
--- a/tests/files/packet_ubuntu22-aio-docker.yml
+++ b/tests/files/packet_ubuntu22-aio-docker.yml
@@ -15,3 +15,4 @@ enable_nodelocaldns: False
 container_manager: docker
 etcd_deployment_type: docker
 resolvconf_mode: docker_dns
+docker_repo_key_keyring: /etc/apt/trusted.gpg.d/docker.gpg