From 2cb573238e10d81d97923394ac4f5932655171bf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Skyler=20M=C3=A4ntysaari?= <samip5@users.noreply.github.com>
Date: Mon, 17 Jul 2023 20:34:31 +0300
Subject: [PATCH 01/12] Ansible: IPv6 :rocket:

---
 .../inventory/group_vars/kubernetes/supplemental.yaml.j2   | 4 ++++
 .../ansible/inventory/group_vars/master/main.yaml.j2       | 4 ++++
 .../ansible/inventory/group_vars/worker/main.yaml.j2       | 4 ++++
 .../playbooks/templates/custom-cilium-helmchart.yaml.j2.j2 | 7 +++++++
 .../templates/custom-coredns-helmchart.yaml.j2.j2          | 2 +-
 bootstrap/vars/config.sample.yaml                          | 5 +++++
 6 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/bootstrap/templates/ansible/inventory/group_vars/kubernetes/supplemental.yaml.j2 b/bootstrap/templates/ansible/inventory/group_vars/kubernetes/supplemental.yaml.j2
index 6c2bcdc44f4..6cd48e702a8 100644
--- a/bootstrap/templates/ansible/inventory/group_vars/kubernetes/supplemental.yaml.j2
+++ b/bootstrap/templates/ansible/inventory/group_vars/kubernetes/supplemental.yaml.j2
@@ -4,5 +4,9 @@ github_username: "{{ bootstrap_github_username }}"
 coredns_addr: "{{ bootstrap_coredns_addr }}"
 kube_vip_addr: "{{ bootstrap_kube_vip_addr }}"
 cluster_cidr: "{{ bootstrap_cluster_cidr }}"
+cluster_cidr_v4: "{{ cluster_cidr.split(',')[0] | regex_replace('/.*', '') }}"
+cluster_cidr_v6: "{{ cluster_cidr.split(',')[1] | default('') | regex_replace('/.*', '') }}"
 service_cidr: "{{ bootstrap_service_cidr }}"
+service_cidr_v4: "{{ service_cidr.split(',')[0] | regex_replace('/.*', '') }}"
+service_cidr_v6: "{{ service_cidr.split(',')[1] | default('') | regex_replace('/.*', '') }}"
 node_cidr: "{{ bootstrap_node_cidr }}"
diff --git a/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2 b/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2
index 481be05d515..12dd1e5de00 100644
--- a/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2
+++ b/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2
@@ -4,7 +4,11 @@
 
 k3s_control_node: true
 k3s_server:
+  {% if ipv6_enabled %}
+  node-ip: "{% raw %}{{ ansible_host }},{{ ansible_default_ipv6.address }}{% endraw %}"
+  {% else %}
   node-ip: "{% raw %}{{ ansible_host }}{% endraw %}"
+  {% endif %}
   tls-san:
     - "{% raw %}{{ kube_vip_addr }}{% endraw %}"
   docker: false
diff --git a/bootstrap/templates/ansible/inventory/group_vars/worker/main.yaml.j2 b/bootstrap/templates/ansible/inventory/group_vars/worker/main.yaml.j2
index 291998dec09..ed8052f4859 100644
--- a/bootstrap/templates/ansible/inventory/group_vars/worker/main.yaml.j2
+++ b/bootstrap/templates/ansible/inventory/group_vars/worker/main.yaml.j2
@@ -4,4 +4,8 @@
 
 k3s_control_node: false
 k3s_agent:
+  {% if ipv6_enabled %}
+  node-ip: "{% raw %}{{ ansible_host }},{{ ansible_default_ipv6.address }}{% endraw %}"
+  {% else %}
   node-ip: "{% raw %}{{ ansible_host }}{% endraw %}"
+  {% endif %}
diff --git a/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2 b/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2
index 6bcfe6ebf12..bb45bea7829 100644
--- a/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2
+++ b/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2
@@ -30,7 +30,14 @@ spec:
       enabled: false
     ipam:
       mode: kubernetes
+    {% if ipv6_enabled %}
+    ipv4NativeRoutingCIDR: "{% raw %}{{ cluster_cidr_v4 }}{% endraw %}"
+    ipv6NativeRoutingCIDR: "{% raw %}{{ cluster_cidr_v6 }}{% endraw %}"
+    ipv6:
+      enabled: true
+    {% else %}
     ipv4NativeRoutingCIDR: "{% raw %}{{ k3s_server['cluster-cidr'] }}{% endraw %}"
+    {% endif %}
     k8sServiceHost: "{% raw %}{{ kube_vip_addr }}{% endraw %}"
     k8sServicePort: 6443
     kubeProxyReplacement: strict
diff --git a/bootstrap/templates/ansible/playbooks/templates/custom-coredns-helmchart.yaml.j2.j2 b/bootstrap/templates/ansible/playbooks/templates/custom-coredns-helmchart.yaml.j2.j2
index 1c4552d1b6a..f985dde1dba 100644
--- a/bootstrap/templates/ansible/playbooks/templates/custom-coredns-helmchart.yaml.j2.j2
+++ b/bootstrap/templates/ansible/playbooks/templates/custom-coredns-helmchart.yaml.j2.j2
@@ -19,7 +19,7 @@ spec:
     service:
       name: kube-dns
       # Choose the 10th IP address from the start of the service-cidr
-      clusterIP: "{% raw %}{{ k3s_server['service-cidr'] | ansible.utils.nthhost(10) }}{% endraw %}"
+      clusterIP: "{% raw %}{{ service_cidr_v4 | ansible.utils.nthhost(10) }}{% endraw %}"
     serviceAccount:
       create: true
     deployment:
diff --git a/bootstrap/vars/config.sample.yaml b/bootstrap/vars/config.sample.yaml
index 97f67a964b1..39f3d4eb95c 100644
--- a/bootstrap/vars/config.sample.yaml
+++ b/bootstrap/vars/config.sample.yaml
@@ -45,6 +45,11 @@ bootstrap_ingress_nginx_addr:
 # Choose your cluster and service cidrs, leave these unchanged unless you know what you are doing
 bootstrap_cluster_cidr: 10.42.0.0/16
 bootstrap_service_cidr: 10.43.0.0/16
+ipv6_enabled: false
+# If you want IPv6 enabled, please uncomment the below vars and comment the ones above
+# bootstrap_cluster_cidr: 10.42.0.0/16,fd7f:8f5:e87c:a::/64
+# bootstrap_service_cidr: 10.43.0.0/16,fd7f:8f5:e87c:e::/112
+# ipv6_enabled: true
 
 # Node information
 bootstrap_nodes:

From dd920b3295d5f1426f75a02fcb8549bea89e5bd4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Skyler=20M=C3=A4ntysaari?= <samip5@users.noreply.github.com>
Date: Mon, 17 Jul 2023 21:18:22 +0300
Subject: [PATCH 02/12] Ansible: IPv6 net validation :rocket:

---
 bootstrap/tasks/validation/net.yaml | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/bootstrap/tasks/validation/net.yaml b/bootstrap/tasks/validation/net.yaml
index bad577d2982..880f0f8cbbf 100644
--- a/bootstrap/tasks/validation/net.yaml
+++ b/bootstrap/tasks/validation/net.yaml
@@ -15,13 +15,15 @@
 
 - name: Verify cluster CIDR
   ansible.builtin.assert:
-    that: ["{{ bootstrap_cluster_cidr is ansible.utils.ipv4 }}"]
+    that:
+     - "{{ bootstrap_cluster_cidr | regex_search('^(?:[0-9]{1,3}\\.){3}[0-9]{1,3}\\/[0-9]{1,2}$') is match('^.*$') or bootstrap_cluster_cidr | regex_search('^([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}\\/[0-9]{1,3}$') is match('^.*$') }}"
     success_msg: "Cluster CIDR {{ bootstrap_cluster_cidr }} is valid."
     fail_msg: "Cluster CIDR {{ bootstrap_cluster_cidr }} is invalid."
 
 - name: Verify service CIDR
   ansible.builtin.assert:
-    that: ["{{ bootstrap_service_cidr is ansible.utils.ipv4 }}"]
+    that:
+     - "{{ bootstrap_service_cidr | regex_search('^(?:[0-9]{1,3}\\.){3}[0-9]{1,3}\\/[0-9]{1,2}$') is match('^.*$') or bootstrap_service_cidr | regex_search('^([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}\\/[0-9]{1,3}$') is match('^.*$') }}"
     success_msg: "Service CIDR {{ bootstrap_service_cidr }} is valid."
     fail_msg: "Service CIDR {{ bootstrap_service_cidr }} is invalid."
 
@@ -92,3 +94,11 @@
   loop: "{{ bootstrap_nodes.master + bootstrap_nodes.worker | default([]) }}"
   loop_control:
      label: "{{ item.address }}"
+
+- Name: Verify that nodes have an IPv6 address
+  ansible.builtin.assert:
+    that: ["{{ ansible_default_ipv6.address is ansible.utils.ipv6 }}"]
+    success_msg: "Node address {{ ansible_default_ipv6.address }} is valid IPv6."
+    fail_msg: "Node IPv6 address is not valid or it's missing from node."
+    quiet: true
+  when: ipv6_enabled

From 22c2bd0e676de10f763a85b4325275288fc6268e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Skyler=20M=C3=A4ntysaari?= <samip5@users.noreply.github.com>
Date: Mon, 17 Jul 2023 22:58:38 +0300
Subject: [PATCH 03/12] Address PR comments about net validation

---
 bootstrap/tasks/validation/net.yaml | 56 +++++++++++++++++++----------
 1 file changed, 38 insertions(+), 18 deletions(-)

diff --git a/bootstrap/tasks/validation/net.yaml b/bootstrap/tasks/validation/net.yaml
index 880f0f8cbbf..7f19bb7ab8e 100644
--- a/bootstrap/tasks/validation/net.yaml
+++ b/bootstrap/tasks/validation/net.yaml
@@ -13,19 +13,47 @@
     success_msg: "Node CIDR {{ bootstrap_node_cidr }} is valid."
     fail_msg: "Node CIDR {{ bootstrap_node_cidr }} is invalid."
 
-- name: Verify cluster CIDR
+- name: Verify cluster CIDR is ipv4 OR ipv6
+  when: not bootstrap_ipv6_enabled | default(false)
   ansible.builtin.assert:
-    that:
-     - "{{ bootstrap_cluster_cidr | regex_search('^(?:[0-9]{1,3}\\.){3}[0-9]{1,3}\\/[0-9]{1,2}$') is match('^.*$') or bootstrap_cluster_cidr | regex_search('^([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}\\/[0-9]{1,3}$') is match('^.*$') }}"
-    success_msg: "Cluster CIDR {{ bootstrap_cluster_cidr }} is valid."
-    fail_msg: "Cluster CIDR {{ bootstrap_cluster_cidr }} is invalid."
+    that: bootstrap_cluster_cidr is ansible.utils.ipv4 or bootstrap_cluster_cidr is ansible.utils.ipv6
+    success_msg: Cluster CIDR {{ bootstrap_cluster_cidr }} is valid.
+    fail_msg: Cluster CIDR {{ bootstrap_cluster_cidr }} is invalid.
 
-- name: Verify service CIDR
+- name: Verify service CIDR is ipv4 OR ipv6
+  when: not bootstrap_ipv6_enabled | default(false)
   ansible.builtin.assert:
-    that:
-     - "{{ bootstrap_service_cidr | regex_search('^(?:[0-9]{1,3}\\.){3}[0-9]{1,3}\\/[0-9]{1,2}$') is match('^.*$') or bootstrap_service_cidr | regex_search('^([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}\\/[0-9]{1,3}$') is match('^.*$') }}"
-    success_msg: "Service CIDR {{ bootstrap_service_cidr }} is valid."
-    fail_msg: "Service CIDR {{ bootstrap_service_cidr }} is invalid."
+    that: bootstrap_service_cidr is ansible.utils.ipv4 or bootstrap_service_cidr is ansible.utils.ipv6
+    success_msg: Service CIDR {{ bootstrap_service_cidr }} is valid.
+    fail_msg: Service CIDR {{ bootstrap_service_cidr }} is invalid.
+
+- name: Verify cluster CIDR is ipv4 AND ipv6
+  when: bootstrap_ipv6_enabled | default(false)
+  ansible.builtin.assert:
+    that: >
+      (
+        bootstrap_cluster_cidr.split(',')[0] is ansible.utils.ipv4 or
+          bootstrap_cluster_cidr.split(',')[1] is ansible.utils.ipv4
+      ) and (
+        bootstrap_cluster_cidr.split(',')[1] is ansible.utils.ipv6 or
+          bootstrap_cluster_cidr.split(',')[0] is ansible.utils.ipv6
+      )
+    success_msg: Cluster CIDR {{ bootstrap_cluster_cidr }} is valid.
+    fail_msg: Cluster CIDR {{ bootstrap_cluster_cidr }} is invalid.
+
+- name: Verify service CIDR is ipv4 AND ipv6
+  when: bootstrap_ipv6_enabled | default(false)
+  ansible.builtin.assert:
+    that: >
+      (
+        bootstrap_service_cidr.split(',')[0] is ansible.utils.ipv4 or
+          bootstrap_service_cidr.split(',')[1] is ansible.utils.ipv4
+      ) and (
+        bootstrap_service_cidr.split(',')[1] is ansible.utils.ipv6 or
+          bootstrap_service_cidr.split(',')[0] is ansible.utils.ipv6
+      )
+    success_msg: Cluster CIDR {{ bootstrap_service_cidr }} is valid.
+    fail_msg: Cluster CIDR {{ bootstrap_service_cidr }} is invalid.
 
 - name: Verify k8s_gateway
   ansible.builtin.assert:
@@ -94,11 +122,3 @@
   loop: "{{ bootstrap_nodes.master + bootstrap_nodes.worker | default([]) }}"
   loop_control:
      label: "{{ item.address }}"
-
-- Name: Verify that nodes have an IPv6 address
-  ansible.builtin.assert:
-    that: ["{{ ansible_default_ipv6.address is ansible.utils.ipv6 }}"]
-    success_msg: "Node address {{ ansible_default_ipv6.address }} is valid IPv6."
-    fail_msg: "Node IPv6 address is not valid or it's missing from node."
-    quiet: true
-  when: ipv6_enabled

From d97f9d67259f0f6e7ffdec8891e311acaa0e7cde Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Skyler=20M=C3=A4ntysaari?= <samip5@users.noreply.github.com>
Date: Mon, 17 Jul 2023 23:02:46 +0300
Subject: [PATCH 04/12] Address PR comments

---
 .../ansible/inventory/group_vars/master/main.yaml.j2       | 6 +++---
 .../ansible/inventory/group_vars/worker/main.yaml.j2       | 6 +++---
 .../playbooks/templates/custom-cilium-helmchart.yaml.j2.j2 | 7 ++++++-
 3 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2 b/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2
index 12dd1e5de00..f2e5b5fab00 100644
--- a/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2
+++ b/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2
@@ -4,11 +4,11 @@
 
 k3s_control_node: true
 k3s_server:
-  {% if ipv6_enabled %}
+{% if bootstrap_ipv6_enabled %}
   node-ip: "{% raw %}{{ ansible_host }},{{ ansible_default_ipv6.address }}{% endraw %}"
-  {% else %}
+{% else %}
   node-ip: "{% raw %}{{ ansible_host }}{% endraw %}"
-  {% endif %}
+{% endif %}
   tls-san:
     - "{% raw %}{{ kube_vip_addr }}{% endraw %}"
   docker: false
diff --git a/bootstrap/templates/ansible/inventory/group_vars/worker/main.yaml.j2 b/bootstrap/templates/ansible/inventory/group_vars/worker/main.yaml.j2
index ed8052f4859..11a4a473812 100644
--- a/bootstrap/templates/ansible/inventory/group_vars/worker/main.yaml.j2
+++ b/bootstrap/templates/ansible/inventory/group_vars/worker/main.yaml.j2
@@ -4,8 +4,8 @@
 
 k3s_control_node: false
 k3s_agent:
-  {% if ipv6_enabled %}
+{% if bootstrap_ipv6_enabled %}
   node-ip: "{% raw %}{{ ansible_host }},{{ ansible_default_ipv6.address }}{% endraw %}"
-  {% else %}
+{% else %}
   node-ip: "{% raw %}{{ ansible_host }}{% endraw %}"
-  {% endif %}
+{% endif %}
diff --git a/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2 b/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2
index bb45bea7829..e9fdcbfecc1 100644
--- a/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2
+++ b/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2
@@ -30,7 +30,7 @@ spec:
       enabled: false
     ipam:
       mode: kubernetes
-    {% if ipv6_enabled %}
+    {% if bootstrap_ipv6_enabled %}
     ipv4NativeRoutingCIDR: "{% raw %}{{ cluster_cidr_v4 }}{% endraw %}"
     ipv6NativeRoutingCIDR: "{% raw %}{{ cluster_cidr_v6 }}{% endraw %}"
     ipv6:
@@ -42,12 +42,17 @@ spec:
     k8sServicePort: 6443
     kubeProxyReplacement: strict
     kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256
+    {% if bootstrap_ipv6_enabled %}
+    l2announcements:
+      enabled: false
+    {% else %}
     # https://github.com/cilium/cilium/issues/26586
     l2announcements:
       enabled: true
       leaseDuration: 120s
       leaseRenewDeadline: 60s
       leaseRetryPeriod: 1s
+    {% endif %}
     loadBalancer:
       algorithm: maglev
       mode: dsr

From a68e9c2375a2ac298ec3b54b906a60ed353991d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Skyler=20M=C3=A4ntysaari?= <samip5@users.noreply.github.com>
Date: Mon, 17 Jul 2023 23:03:38 +0300
Subject: [PATCH 05/12] Update bootstrap/vars/config.sample.yaml

Co-authored-by: Devin Buhl <onedr0p@users.noreply.github.com>
---
 bootstrap/vars/config.sample.yaml | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/bootstrap/vars/config.sample.yaml b/bootstrap/vars/config.sample.yaml
index 39f3d4eb95c..077789d4ac6 100644
--- a/bootstrap/vars/config.sample.yaml
+++ b/bootstrap/vars/config.sample.yaml
@@ -42,14 +42,15 @@ bootstrap_kube_vip_addr:
 bootstrap_k8s_gateway_addr:
 # The Load balancer IP for ingress-nginx, choose an available IP in your nodes network that is not being used
 bootstrap_ingress_nginx_addr:
-# Choose your cluster and service cidrs, leave these unchanged unless you know what you are doing
+
+# Keep the next three options default unless you know what you are doing
+# (Advanced) Enable ipv6
+bootstrap_ipv6_enabled: false
+# (Advanced) For ipv6 use format 10.42.0.0/16,fd7f:8f5:e87c:a::/64
 bootstrap_cluster_cidr: 10.42.0.0/16
+# (Advanced) For ipv6 use format 10.43.0.0/16,fd7f:8f5:e87c:e::/112
 bootstrap_service_cidr: 10.43.0.0/16
-ipv6_enabled: false
-# If you want IPv6 enabled, please uncomment the below vars and comment the ones above
-# bootstrap_cluster_cidr: 10.42.0.0/16,fd7f:8f5:e87c:a::/64
-# bootstrap_service_cidr: 10.43.0.0/16,fd7f:8f5:e87c:e::/112
-# ipv6_enabled: true
+
 
 # Node information
 bootstrap_nodes:

From 17e66eed88112c76493613d369d5c9b3f42ff508 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Skyler=20M=C3=A4ntysaari?= <samip5@users.noreply.github.com>
Date: Mon, 17 Jul 2023 23:06:36 +0300
Subject: [PATCH 06/12] Make a note about /60 IPv6 block

---
 bootstrap/vars/config.sample.yaml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/bootstrap/vars/config.sample.yaml b/bootstrap/vars/config.sample.yaml
index 077789d4ac6..dfcb007e4ff 100644
--- a/bootstrap/vars/config.sample.yaml
+++ b/bootstrap/vars/config.sample.yaml
@@ -46,9 +46,10 @@ bootstrap_ingress_nginx_addr:
 # Keep the next three options default unless you know what you are doing
 # (Advanced) Enable ipv6
 bootstrap_ipv6_enabled: false
-# (Advanced) For ipv6 use format 10.42.0.0/16,fd7f:8f5:e87c:a::/64
+# (Advanced) For ipv6 use format 10.42.0.0/16,fd78:c889:47fb:10::/60
+# /60 IPv6 block is enough for 16 nodes
 bootstrap_cluster_cidr: 10.42.0.0/16
-# (Advanced) For ipv6 use format 10.43.0.0/16,fd7f:8f5:e87c:e::/112
+# (Advanced) For ipv6 use format 10.43.0.0/16,fd78:c889:47fb:e0::/112
 bootstrap_service_cidr: 10.43.0.0/16
 
 

From 8b7db5a78bf1d6dbf55542f324afd593ab69dfeb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Skyler=20M=C3=A4ntysaari?= <samip5@users.noreply.github.com>
Date: Mon, 17 Jul 2023 23:13:08 +0300
Subject: [PATCH 07/12] Jinja things..

---
 .../templates/ansible/inventory/group_vars/master/main.yaml.j2   | 1 +
 .../templates/ansible/inventory/group_vars/worker/main.yaml.j2   | 1 +
 .../playbooks/templates/custom-cilium-helmchart.yaml.j2.j2       | 1 +
 3 files changed, 3 insertions(+)

diff --git a/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2 b/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2
index f2e5b5fab00..fb913ae987b 100644
--- a/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2
+++ b/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2
@@ -1,3 +1,4 @@
+#jinja2: trim_blocks: True, lstrip_blocks: True
 ---
 # https://rancher.com/docs/k3s/latest/en/installation/install-options/server-config/
 # https://github.com/PyratLabs/ansible-role-k3s
diff --git a/bootstrap/templates/ansible/inventory/group_vars/worker/main.yaml.j2 b/bootstrap/templates/ansible/inventory/group_vars/worker/main.yaml.j2
index 11a4a473812..6801109c942 100644
--- a/bootstrap/templates/ansible/inventory/group_vars/worker/main.yaml.j2
+++ b/bootstrap/templates/ansible/inventory/group_vars/worker/main.yaml.j2
@@ -1,3 +1,4 @@
+#jinja2: trim_blocks: True, lstrip_blocks: True
 ---
 # https://rancher.com/docs/k3s/latest/en/installation/install-options/agent-config/
 # https://github.com/PyratLabs/ansible-role-k3s
diff --git a/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2 b/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2
index e9fdcbfecc1..bf37bd51efe 100644
--- a/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2
+++ b/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2
@@ -1,3 +1,4 @@
+#jinja2: trim_blocks: True, lstrip_blocks: True
 ---
 # https://docs.k3s.io/helm
 apiVersion: helm.cattle.io/v1

From b7028270bb919064a580caa527e2be7a825bd3b0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Skyler=20M=C3=A4ntysaari?= <samip5@users.noreply.github.com>
Date: Mon, 17 Jul 2023 23:32:47 +0300
Subject: [PATCH 08/12] Apply suggestions from code review

Co-authored-by: Devin Buhl <onedr0p@users.noreply.github.com>
---
 .../playbooks/templates/custom-cilium-helmchart.yaml.j2.j2    | 4 ++--
 .../playbooks/templates/custom-coredns-helmchart.yaml.j2.j2   | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2 b/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2
index bf37bd51efe..185162e5beb 100644
--- a/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2
+++ b/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2
@@ -32,8 +32,8 @@ spec:
     ipam:
       mode: kubernetes
     {% if bootstrap_ipv6_enabled %}
-    ipv4NativeRoutingCIDR: "{% raw %}{{ cluster_cidr_v4 }}{% endraw %}"
-    ipv6NativeRoutingCIDR: "{% raw %}{{ cluster_cidr_v6 }}{% endraw %}"
+    ipv4NativeRoutingCIDR: "{% raw %}{{ k3s_server['cluster-cidr'].split(',')[0] }}{% endraw %}"
+    ipv6NativeRoutingCIDR: "{% raw %}{{ k3s_server['cluster-cidr'].split(',')[1] }}{% endraw %}"
     ipv6:
       enabled: true
     {% else %}
diff --git a/bootstrap/templates/ansible/playbooks/templates/custom-coredns-helmchart.yaml.j2.j2 b/bootstrap/templates/ansible/playbooks/templates/custom-coredns-helmchart.yaml.j2.j2
index f985dde1dba..1693f5a7f10 100644
--- a/bootstrap/templates/ansible/playbooks/templates/custom-coredns-helmchart.yaml.j2.j2
+++ b/bootstrap/templates/ansible/playbooks/templates/custom-coredns-helmchart.yaml.j2.j2
@@ -19,7 +19,7 @@ spec:
     service:
       name: kube-dns
       # Choose the 10th IP address from the start of the service-cidr
-      clusterIP: "{% raw %}{{ service_cidr_v4 | ansible.utils.nthhost(10) }}{% endraw %}"
+      clusterIP: "{% raw %}{{ k3s_server['service-cidr'].split(',')[0] | ansible.utils.nthhost(10) }}{% endraw %}"
     serviceAccount:
       create: true
     deployment:

From 4da4b16531b96df1b2cf3e1ddfc2a6a6fb0d035f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Skyler=20M=C3=A4ntysaari?= <samip5@users.noreply.github.com>
Date: Mon, 17 Jul 2023 23:33:48 +0300
Subject: [PATCH 09/12] Yeet

---
 .../inventory/group_vars/kubernetes/supplemental.yaml.j2      | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/bootstrap/templates/ansible/inventory/group_vars/kubernetes/supplemental.yaml.j2 b/bootstrap/templates/ansible/inventory/group_vars/kubernetes/supplemental.yaml.j2
index 6cd48e702a8..6c2bcdc44f4 100644
--- a/bootstrap/templates/ansible/inventory/group_vars/kubernetes/supplemental.yaml.j2
+++ b/bootstrap/templates/ansible/inventory/group_vars/kubernetes/supplemental.yaml.j2
@@ -4,9 +4,5 @@ github_username: "{{ bootstrap_github_username }}"
 coredns_addr: "{{ bootstrap_coredns_addr }}"
 kube_vip_addr: "{{ bootstrap_kube_vip_addr }}"
 cluster_cidr: "{{ bootstrap_cluster_cidr }}"
-cluster_cidr_v4: "{{ cluster_cidr.split(',')[0] | regex_replace('/.*', '') }}"
-cluster_cidr_v6: "{{ cluster_cidr.split(',')[1] | default('') | regex_replace('/.*', '') }}"
 service_cidr: "{{ bootstrap_service_cidr }}"
-service_cidr_v4: "{{ service_cidr.split(',')[0] | regex_replace('/.*', '') }}"
-service_cidr_v6: "{{ service_cidr.split(',')[1] | default('') | regex_replace('/.*', '') }}"
 node_cidr: "{{ bootstrap_node_cidr }}"

From c961e584fb5e733fda744ca8246fd5e0ccc40864 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Skyler=20M=C3=A4ntysaari?= <samip5@users.noreply.github.com>
Date: Mon, 17 Jul 2023 23:34:05 +0300
Subject: [PATCH 10/12] Update bootstrap/tasks/validation/net.yaml

Co-authored-by: Devin Buhl <onedr0p@users.noreply.github.com>
---
 bootstrap/tasks/validation/net.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bootstrap/tasks/validation/net.yaml b/bootstrap/tasks/validation/net.yaml
index 7f19bb7ab8e..7203fc7781e 100644
--- a/bootstrap/tasks/validation/net.yaml
+++ b/bootstrap/tasks/validation/net.yaml
@@ -52,8 +52,8 @@
         bootstrap_service_cidr.split(',')[1] is ansible.utils.ipv6 or
           bootstrap_service_cidr.split(',')[0] is ansible.utils.ipv6
       )
-    success_msg: Cluster CIDR {{ bootstrap_service_cidr }} is valid.
-    fail_msg: Cluster CIDR {{ bootstrap_service_cidr }} is invalid.
+    success_msg: Service CIDR {{ bootstrap_service_cidr }} is valid.
+    fail_msg: Service CIDR {{ bootstrap_service_cidr }} is invalid.
 
 - name: Verify k8s_gateway
   ansible.builtin.assert:

From cf15efeb1954a9221262466585246b24947d242f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Skyler=20M=C3=A4ntysaari?= <samip5@users.noreply.github.com>
Date: Mon, 17 Jul 2023 23:38:26 +0300
Subject: [PATCH 11/12] Node addresses should be IPv4

Ansible will connect over IPv4
---
 bootstrap/tasks/validation/net.yaml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/bootstrap/tasks/validation/net.yaml b/bootstrap/tasks/validation/net.yaml
index 7203fc7781e..1102a9405e5 100644
--- a/bootstrap/tasks/validation/net.yaml
+++ b/bootstrap/tasks/validation/net.yaml
@@ -102,6 +102,16 @@
   loop_control:
      label: "{{ item.address }}"
 
+- name: Verify nodes are ipv4
+  ansible.builtin.assert:
+    that: item.address is ansible.utils.ipv4
+    success_msg: Node address {{ item.address }} is valid.
+    fail_msg: Node address {{ item.address }} is invalid.
+    quiet: true
+  loop: "{{ bootstrap_nodes.master + bootstrap_nodes.worker | default([]) }}"
+  loop_control:
+    label: "{{ item.address }}"
+
 - name: Verify nodes are in node CIDR
   ansible.builtin.assert:
     that: ["{{ bootstrap_node_cidr | ansible.utils.network_in_usable(item.address) }}"]

From 7888a8b073325239fbdf7948913378804ed2fee8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Skyler=20M=C3=A4ntysaari?= <samip5@users.noreply.github.com>
Date: Mon, 17 Jul 2023 23:39:29 +0300
Subject: [PATCH 12/12] Apply suggestions from code review

Co-authored-by: Devin Buhl <onedr0p@users.noreply.github.com>
---
 .../ansible/inventory/group_vars/master/main.yaml.j2          | 2 +-
 .../ansible/inventory/group_vars/worker/main.yaml.j2          | 2 +-
 .../playbooks/templates/custom-cilium-helmchart.yaml.j2.j2    | 4 ++--
 .../playbooks/templates/custom-coredns-helmchart.yaml.j2.j2   | 4 ++++
 4 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2 b/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2
index fb913ae987b..70764f63299 100644
--- a/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2
+++ b/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2
@@ -5,7 +5,7 @@
 
 k3s_control_node: true
 k3s_server:
-{% if bootstrap_ipv6_enabled %}
+{% if bootstrap_ipv6_enabled | default(false) %}
   node-ip: "{% raw %}{{ ansible_host }},{{ ansible_default_ipv6.address }}{% endraw %}"
 {% else %}
   node-ip: "{% raw %}{{ ansible_host }}{% endraw %}"
diff --git a/bootstrap/templates/ansible/inventory/group_vars/worker/main.yaml.j2 b/bootstrap/templates/ansible/inventory/group_vars/worker/main.yaml.j2
index 6801109c942..687cf0dc3ac 100644
--- a/bootstrap/templates/ansible/inventory/group_vars/worker/main.yaml.j2
+++ b/bootstrap/templates/ansible/inventory/group_vars/worker/main.yaml.j2
@@ -5,7 +5,7 @@
 
 k3s_control_node: false
 k3s_agent:
-{% if bootstrap_ipv6_enabled %}
+{% if bootstrap_ipv6_enabled | default(false) %}
   node-ip: "{% raw %}{{ ansible_host }},{{ ansible_default_ipv6.address }}{% endraw %}"
 {% else %}
   node-ip: "{% raw %}{{ ansible_host }}{% endraw %}"
diff --git a/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2 b/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2
index 185162e5beb..ee72855a588 100644
--- a/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2
+++ b/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2
@@ -31,7 +31,7 @@ spec:
       enabled: false
     ipam:
       mode: kubernetes
-    {% if bootstrap_ipv6_enabled %}
+    {% if bootstrap_ipv6_enabled | default(false) %}
     ipv4NativeRoutingCIDR: "{% raw %}{{ k3s_server['cluster-cidr'].split(',')[0] }}{% endraw %}"
     ipv6NativeRoutingCIDR: "{% raw %}{{ k3s_server['cluster-cidr'].split(',')[1] }}{% endraw %}"
     ipv6:
@@ -43,7 +43,7 @@ spec:
     k8sServicePort: 6443
     kubeProxyReplacement: strict
     kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256
-    {% if bootstrap_ipv6_enabled %}
+    {% if bootstrap_ipv6_enabled | default(false) %}
     l2announcements:
       enabled: false
     {% else %}
diff --git a/bootstrap/templates/ansible/playbooks/templates/custom-coredns-helmchart.yaml.j2.j2 b/bootstrap/templates/ansible/playbooks/templates/custom-coredns-helmchart.yaml.j2.j2
index 1693f5a7f10..17cd15ddfbb 100644
--- a/bootstrap/templates/ansible/playbooks/templates/custom-coredns-helmchart.yaml.j2.j2
+++ b/bootstrap/templates/ansible/playbooks/templates/custom-coredns-helmchart.yaml.j2.j2
@@ -19,7 +19,11 @@ spec:
     service:
       name: kube-dns
       # Choose the 10th IP address from the start of the service-cidr
+      {% if bootstrap_ipv6_enabled | default(false) %}
       clusterIP: "{% raw %}{{ k3s_server['service-cidr'].split(',')[0] | ansible.utils.nthhost(10) }}{% endraw %}"
+      {% else %}
+      clusterIP: "{% raw %}{{ k3s_server['service-cidr'] | ansible.utils.nthhost(10) }}{% endraw %}"
+      {% endif %}
     serviceAccount:
       create: true
     deployment: