SYS-626 ubuntu 24.04 and ansible updates for k8s 1.31 (#169)

* SYS-626 ubuntu 24.04 and ansible updates for k8s 1.31

ansible/README.md

@@ -1,7 +1,7 @@
-## Ansible for Kubernetes and Docker Swarm
+## Ansible for Kubernetes
 
-This directory contains playbooks to set up Kubernetes or Docker swarm
-nodes in a secure way, by techniques recommended in Kubernetes and
+This directory contains playbooks to set up Kubernetes nodes
+in a secure way, by techniques recommended in Kubernetes and
 Docker documentation plus numerous 3rd-party best-practices
 recommendations. It also has tools to configure LVM for LUKS encrypted
 disk volumes.
@@ -18,8 +18,7 @@ effort.
   a self-signed API cert by storing the same cert as ca-root.pem.
 * Define variables as listed below, in your group_vars directory.
 * Define host inventory as cluster nodes and managers, in hosts. For
-  kubernetes, use group names k8s_master and k8s_nodes; for swarm,
-  use group swarm_managers and swarm_nodes.
+  kubernetes, use group names k8s_cplane and k8s_nodes.
 * Add any ssh public keys for your user(s) into files/keys/ssh_public_keys,
   with file names <user>-<keyname>.pub; these will be added to
   ~<user>/.ssh/authorized_keys.
@@ -30,18 +29,11 @@ This includes a Makefile with the following targets, to be run in the
 following (approximate) order for kubernetes (see detailed readme in k8s
 directory):
 ```
-make k8s-master
+make k8s-cplane
 cd ../k8s ; make install
 make k8s-node
 ```
 
-For swarm, do this:
-```
-make test
-make volume-setup
-make node-setup join-swarm
-```
-
 ### Variables
 Local variables:
 

ansible/hosts-lab

@@ -1,7 +1,7 @@
 [fileservers]
 k2.ci.net
 
-[k8s_master]
+[k8s_cplane]
 borg1.ci.net
 
 [k8s_nodes]

ansible/k8s-master.yml → ansible/k8s-cplane.yml

@@ -1,28 +1,28 @@
 ---
-# Step 1: set up kubernetes master
+# Step 1: set up kubernetes control plane
 #
 # Usage:
 #  Set up hosts inventory and group_vars as suggested below, then:
 #
-#  ansible-playbook k8s-master.yml
+#  ansible-playbook k8s-cplane.yml
 
-# k8s-master and k8s-node are the first and third steps in cluster setup (with
-# single master and as many worker nodes as desired) which runs
+# k8s-cplane and k8s-node are the first and third steps in cluster setup (with
+# single control plane and as many worker nodes as desired) which runs
 # kubeadm to provide initial installation.  See the Makefile in
 # top-level k8s directory to complete the second step of installation.
 
 # Run the k8s/Makefile and then k8s-node playbook after this.
 
 # Example host inventory, put this in an ansible hosts file:
-#   [k8s_master]
+#   [k8s_cplane]
 #   kubernetes.mydomain.com
 #
 # Pick a separate IP address (optional) for the cluster and define group vars:
-#   put this in group_vars/k8s_master.yml:
+#   put this in group_vars/k8s_cplane.yml:
 #
 # k8s_override:
-#   master: True
-#   master_ip: 192.168.1.10
+#   cplane: True
+#   cplane_vip: 192.168.1.10
 # luks_vg: vg01
 # luks_volumes:
 #   docker:
@@ -42,7 +42,7 @@
 #   mode: static
 #   cluster_ip: 192.168.1.10
 
-- hosts: k8s_master
+- hosts: k8s_cplane
   become: True
   force_handlers: True
   roles:

ansible/k8s-node.yml

@@ -5,12 +5,12 @@
 #
 #  ansible-playbook k8s-node.yml
 
-# k8s-master and k8s-node are the first and third steps in cluster setup (with
-# single master and as many worker nodes as desired) which runs
+# k8s-cplane and k8s-node are the first and third steps in cluster setup (with
+# single control plane and as many worker nodes as desired) which runs
 # kubeadm to provide initial installation.  See the Makefile in
 # top-level k8s directory to complete the second step of installation.
 
-# Run the k8s-master playbook before this one.
+# Run the k8s-cplane playbook before this one.
 
 - hosts: k8s_nodes
   become: True

ansible/roles/docker_node/defaults/main.yml

@@ -9,7 +9,7 @@ docker_defaults:
   apt_repo:
     key: 9DC858229FC7DD38854AE2D88D81803C0EBFCD88
     package_name: docker-ce
-    package_ver: 5:23.0.1-1~ubuntu.22.04~jammy
+    package_ver: 5:27.3.1-1~ubuntu.24.04~noble
     repo: deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable
     url: https://download.docker.com/linux/ubuntu/gpg
   certs:
@@ -32,11 +32,11 @@ docker_defaults:
     log-opts:
       max-size: 50m
       max-file: "3"
-    storage-driver: devicemapper
-    storage-opts:
-    - dm.thinpooldev=/dev/mapper/{{ thinpool_vg_alt }}-thinpool
-    - dm.use_deferred_removal=false
-    - dm.use_deferred_deletion=false
+    storage-driver: overlay2
+    # storage-opts:
+    # - dm.thinpooldev=/dev/mapper/{{ thinpool_vg_alt }}-thinpool
+    # - dm.use_deferred_removal=false
+    # - dm.use_deferred_deletion=false
     tls: True
     tlscacert: /root/certs/ca-root.pem
     tlscert: /root/certs/docker-tls-cert.pem

ansible/roles/docker_node/tasks/docker.yml

@@ -49,42 +49,8 @@
   command: update-ca-certificates
   changed_when: False
 
-- name: Thin pool create
-  lvol:
-    lv: thinpool
-    shrink: False
-    size: "{{ docker.thinpool.size }}"
-    vg: "{{ thinpool_vg }}"
-
-- name: Thin pool meta
-  lvol:
-    lv: thinpoolmeta
-    vg: "{{ thinpool_vg }}"
-    size: "{{ docker.thinpool.meta_size }}"
-
-- name: Convert thinpool
-  command: >
-    lvconvert -y --zero n -c 512K --thinpool {{ thinpool_vg }}/thinpool
-      --poolmetadata {{ thinpool_vg }}/thinpoolmeta
-  args:
-    creates: /dev/mapper/{{ thinpool_vg_alt }}-thinpool_tdata
-
-- name: LVM profile path
-  file:
-    path: /etc/lvm/profile
-    state: directory
-
-- name: Define thinpool autoextend
-  template:
-    dest: /etc/lvm/profile/{{ thinpool_vg }}-thinpool.profile
-    src: thinpool.profile.j2
-  notify: Activate thinpool autoextend
-
-- name: Initiate thinpool autoextend
-  shell: >
-    lvchange --metadataprofile {{ thinpool_vg }}-thinpool {{ thinpool_vg }}/thinpool &&
-    lvs -o+seg_monitor
-  changed_when: False
+- import_tasks: thinpool.yml
+  when: ansible_distribution_version < '24.04'
 
 # The Ubuntu package installer defines ExecStart option that conflicts
 # with our daemon.json. Need to suppress daemon startup until after
@@ -186,6 +152,12 @@
     mode: 0755
     url: https://github.com/docker/compose/releases/download/{{
       docker.compose.version }}/docker-compose-Linux-x86_64
+  when: ansible_distribution_version < '24.04'
+
+- name: Install docker-compose from repo
+  apt:
+    name: docker-compose
+  when: ansible_distribution_version >= '24.04'
 
 - name: Sysctl tuning parameters
   sysctl:

ansible/roles/docker_node/tasks/repos.yml

@@ -5,14 +5,6 @@
     filename: ubuntu
   with_items: "{{ ubuntu_repos }}"
 
-
-# TODO remove this at next k8s and ubuntu update (24.04)
-- name: Remove stale k8s repo
-  apt_repository:
-    filename: k8s
-    repo: "{{ k8s.apt_repo.repo }}"
-    state: absent
-
 - name: Docker repo key
   get_url:
     url: "{{ docker.apt_repo.url }}"

ansible/roles/docker_node/tasks/thinpool.yml

@@ -0,0 +1,36 @@
+- name: Thin pool create
+  lvol:
+    lv: thinpool
+    shrink: False
+    size: "{{ docker.thinpool.size }}"
+    vg: "{{ thinpool_vg }}"
+
+- name: Thin pool meta
+  lvol:
+    lv: thinpoolmeta
+    vg: "{{ thinpool_vg }}"
+    size: "{{ docker.thinpool.meta_size }}"
+
+- name: Convert thinpool
+  command: >
+    lvconvert -y --zero n -c 512K --thinpool {{ thinpool_vg }}/thinpool
+      --poolmetadata {{ thinpool_vg }}/thinpoolmeta
+  args:
+    creates: /dev/mapper/{{ thinpool_vg_alt }}-thinpool_tdata
+
+- name: LVM profile path
+  file:
+    path: /etc/lvm/profile
+    state: directory
+
+- name: Define thinpool autoextend
+  template:
+    dest: /etc/lvm/profile/{{ thinpool_vg }}-thinpool.profile
+    src: thinpool.profile.j2
+  notify: Activate thinpool autoextend
+
+- name: Initiate thinpool autoextend
+  shell: >
+    lvchange --metadataprofile {{ thinpool_vg }}-thinpool {{ thinpool_vg }}/thinpool &&
+    lvs -o+seg_monitor
+  changed_when: False

ansible/roles/kubernetes/defaults/main.yml

@@ -1,6 +1,6 @@
 ---
 
-k8s_kernel_modules: {}
+k8s_kernel_modules: [ br_netfilter ]
 
 # Generate your own and put in vault
 #   encryption_key: any 32-byte string
@@ -12,26 +12,29 @@ vault_k8s:
 k8s_defaults:
   admin_config: /etc/kubernetes/admin.conf
   apt_repo:
-    # TODO use {{ ansible_distribution_release }}
-    repo: deb [signed-by=/etc/apt/keyrings/kubernetes.asc] http://apt.kubernetes.io/ kubernetes-xenial main
-    url: https://packages.cloud.google.com/apt/doc/apt-key.gpg
+    # TODO parameterize hardcoded 1.31 value
+    repo: deb [signed-by=/etc/apt/keyrings/kubernetes.asc] https://pkgs.k8s.io/core:/stable:/v1.31/deb/ /
+    url: https://pkgs.k8s.io/core:/stable:/v1.31/deb/Release.key
+  cplane_hostip: "{{ hostvars[groups['k8s_cplane'][0]]['ansible_default_ipv4']['address'] | default(groups['k8s_cplane'][0]) }}"
+  cplane_vip: "{{ hostvars[groups['k8s_cplane'][0]]['ansible_default_ipv4']['address'] | default(groups['k8s_cplane'][0]) }}"
   kubeadm_opts: ''
   local_vols: /var/lib/docker/k8s-volumes
-  master: False
-  master_ip: "{{ hostvars[groups['k8s_master'][0]]['ansible_default_ipv4']['address'] | default(groups['k8s_master'][0]) }}"
+  cplane: False
   pod_network: 10.244.0.0/16
+  pod_infra_container_image: registry.k8s.io/pause:3.10
   service:
     enabled: yes
     name: kubelet
     state: restarted
   service_network: 10.96.0.0/12
-  version: 1.25.4
-  cni_version: 1.1.1
+  version: 1.31.2
+  coredns_version: v1.11.3
+  cni_version: 1.5.1
 k8s_override: {}
 k8s: "{{ k8s_defaults | combine(k8s_override) }}"
 
 cri_dockerd:
-  version: 0.2.0
+  version: 0.3.15
 
 oidc:
   client_id: client-not-yet-set
@@ -46,8 +49,8 @@ sysctl:
   vm.swappiness: 0
 
 ubuntu_packages:
-  - kubeadm={{ k8s.version }}-00
-  - kubelet={{ k8s.version }}-00
-  - kubernetes-cni={{ k8s.cni_version }}-00
+  - kubeadm={{ k8s.version }}-*
+  - kubelet={{ k8s.version }}-*
+  - kubernetes-cni={{ k8s.cni_version }}-*
 
 ubuntu_package_additions: []