Bump kubernetes to v1.28.2 (#675)

* bump ansible version to 2.14 * add python3-netaddr binary for kubespray * update list of container images to download * update binaries to download for k8s 1.28.2 * point kubespray to latest master commit * bump nix pkgs version and ansible to 2.16 * use opentofu and ansible 2.15 * update checksum for containerd * update nerdctl checksum * add path to kubespray roles * disable multi networking * add python3-jmespath in the artifact * temp fix for upstream * use jmespath from python3 packages * rebase kubespray v2.24 release * rebase ansible-cassandra with upstream * update k8s dependencies to v1.28.2 in nix * use jmespath-1.0.1 from debian mirror * add jmespath to wsd container * skip multus installation and update docs * re-enable fix_default_router script * fix rabbitmq chart deployment on ubuntu 22.4.3 * use fixes in helm chart wire-server * upgrade cert-manager * print changes in coredns config * Revert "use fixes in helm chart wire-server" This reverts commit 6863399. * debug pods status before installing wire-server * remove jmespath ansible playbooks * [Temp] use the u_id:g_id in webapp chart values * log the pods status on wire-server installation failure * remove runAsUser: 1000 from webapp/teams/accounts page * add variable for fix_default_router script * debug wire-server installation * minor bug with argument * remove upstream changes from values * remove use of fix_default_router script by default * remove additional values from helm example values file * [debug] increase wire-server deploy time in ci * test with latest wire-server from develop branch * cleanup and remove debug statements * fix docs href and kubectl version * use the main wire-server chart repo
wireapp · Feb 13, 2024 · c562b69 · c562b69
1 parent b5bf787
commit c562b69
Show file tree

Hide file tree

Showing 15 changed files with 86 additions and 73 deletions.
diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg
@@ -4,7 +4,7 @@ control_path           = /tmp/ansible-%%r@%%h:%%p
 
 [defaults]
 retry_files_enabled     = False
-roles_path              = ./roles-external:./roles:./roles-external/sft/roles
+roles_path              = ./roles-external:./roles:./roles-external/sft/roles:./roles-external/kubespray/roles
 
 gathering               = smart
 

diff --git a/ansible/inventory/offline/group_vars/all/offline.yml b/ansible/inventory/offline/group_vars/all/offline.yml
@@ -20,10 +20,8 @@ docker_ubuntu_repo_gpgkey: "{{ ubuntu_repos }}/gpg"
 binaries_url: "http://{{ assethost_host }}/binaries"
 nodelocaldns_ip: 10.233.0.10
 
-kube_version: "v1.23.7"
-etcd_version: "v3.5.3"
-
-#container_manager: containerd
+kube_version: "v1.28.2"
+etcd_version: "v3.5.9"
 
 kubeadm_download_url: "{{ binaries_url }}/kubeadm"
 kubectl_download_url: "{{ binaries_url }}/kubectl"

diff --git a/ansible/roles-external/ansible-cassandra b/ansible/roles-external/ansible-cassandra
diff --git a/ansible/roles-external/kubespray b/ansible/roles-external/kubespray
diff --git a/bin/fix_default_router.sh b/bin/fix_default_router.sh
@@ -21,5 +21,6 @@ echo "Updating the configMap coredns -n kube-system"
 kubectl get configmap coredns -n kube-system --output yaml > coredns_config.yaml
 sed -i coredns_config.yaml -e '/^[ ]*forward.*/{N;N;N;d;}' -e "s/^\([ ]*\)cache/\1forward . 127.0.0.53:9999 {\n\1  max_fails 0\n\1}\n\1cache/"
 kubectl apply -f coredns_config.yaml
-
+echo "Printing kubectl get configmap coredns -n kube-system --output yaml after updating"
+kubectl get configmap coredns -n kube-system --output yaml
 sleep 10
diff --git a/bin/offline-cluster.sh b/bin/offline-cluster.sh
@@ -47,9 +47,7 @@ ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/seed-offline-containerd.yml
 ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/sync_time.yml -v
 
 # Run the rest of kubespray. This should bootstrap a kubernetes cluster successfully:
-ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/kubernetes.yml --skip-tags bootstrap-os,preinstall,container-engine
-
-./bin/fix_default_router.sh
+ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/kubernetes.yml --skip-tags bootstrap-os,preinstall,container-engine,multus
 
 # Deploy all other services which don't run in kubernetes.
 ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/cassandra.yml

diff --git a/default.nix b/default.nix
@@ -26,8 +26,9 @@ rec {
   env = pkgs.buildEnv {
     name = "wire-server-deploy";
     paths = with pkgs; [
-      ansible_2_11
+      ansible_2_15
       pythonForAnsible
+      jmespath
       apacheHttpd
       awscli2
       gnumake
@@ -43,7 +44,7 @@ rec {
       moreutils
       skopeo
       sops
-      terraform_1
+      opentofu
       yq
       create-container-dump
       list-helm-containers

diff --git a/nix/pkgs/kubernetes-tools.nix b/nix/pkgs/kubernetes-tools.nix
@@ -3,13 +3,13 @@
 
 buildGoModule rec {
   pname = "kubernetes";
-  version = "1.23.16";
+  version = "1.28.2";
 
   src = fetchFromGitHub {
     owner = "kubernetes";
     repo = "kubernetes";
     rev = "v${version}";
-    hash = "sha256-dLbKzPBMN8w+BA3lQUq6uYr+QoXGMm6SKaWGbYBTH0A=";
+    hash = "sha256-7juoX4nFvQbIIbhTlnIYVUEYUJGwu+aKrpw4ltujjXI=";
   };
 
   vendorSha256 = null;

diff --git a/nix/pkgs/wire-binaries.nix b/nix/pkgs/wire-binaries.nix
@@ -6,16 +6,16 @@ let
   image_arch = "amd64";
 
   # These values are manually kept in sync with:
-  # https://github.com/kubernetes-sigs/kubespray/blob/release-2.20/roles/download/defaults/main.yml
+  # https://github.com/kubernetes-sigs/kubespray/blob/release-2.24/roles/kubespray-defaults/defaults/main/download.yml
   # TODO: Find a better process. Automate this!
-  kube_version = "v1.23.7";
-  etcd_version = "v3.5.3";
-  cni_version = "v1.1.1";
-  calico_version = "v3.23.3";
-  crictl_version = "v1.23.0";
-  runc_version = "v1.1.4";
-  nerdctl_version = "0.22.2";
-  containerd_version = "1.6.8";
+  kube_version = "v1.28.2";
+  etcd_version = "v3.5.9";
+  cni_version = "v1.3.0";
+  calico_version = "v3.26.4";
+  crictl_version = "v1.28.0";
+  runc_version = "v1.1.10";
+  nerdctl_version = "1.7.1";
+  containerd_version = "1.7.11";
 
 
   # Note: If you change a version, replace the checksum with zeros, run « nix-build --no-out-link -A pkgs.wire-binaries », it will complain and give you the right checksum, use that checksum in this file, run it again and it should build without complaining.
@@ -26,57 +26,57 @@ let
     kubelet = fetchurl rec {
       passthru.url = url;
       url = "https://storage.googleapis.com/kubernetes-release/release/${ kube_version }/bin/linux/${ image_arch }/kubelet";
-      sha256 = "518f67200e853253ed6424488d6148476144b6b796ec7c6160cff15769b3e12a";
+      sha256 = "17edb866636f14eceaad58c56eab12af7ab3be3c78400aff9680635d927f1185";
     };
     kubeadm = fetchurl rec {
       passthru.url = url;
       url = "https://storage.googleapis.com/kubernetes-release/release/${ kube_version }/bin/linux/${ image_arch }/kubeadm";
-      sha256 = "d7d863213eeb4791cdbd7c5fd398cf0cc2ef1547b3a74de8285786040f75efd2";
+      sha256 = "6a4808230661c69431143db2e200ea2d021c7f1b1085e6353583075471310d00";
     };
     kubectl = fetchurl rec {
       passthru.url = url;
       url = "https://storage.googleapis.com/kubernetes-release/release/${ kube_version }/bin/linux/${ image_arch }/kubectl";
-      sha256 = "b4c27ad52812ebf3164db927af1a01e503be3fb9dc5ffa058c9281d67c76f66e";
+      sha256 = "c922440b043e5de1afa3c1382f8c663a25f055978cbc6e8423493ec157579ec5";
     };
     crictl = fetchurl rec {
       passthru.url = url;
       url = "https://github.com/kubernetes-sigs/cri-tools/releases/download/${ crictl_version }/crictl-${ crictl_version }-linux-${ image_arch }.tar.gz";
-      sha256 = "b754f83c80acdc75f93aba191ff269da6be45d0fc2d3f4079704e7d1424f1ca8";
+      sha256 = "8dc78774f7cbeaf787994d386eec663f0a3cf24de1ea4893598096cb39ef2508";
     };
     containerd = fetchurl rec {
       passthru.url = url;
       url = "https://github.com/containerd/containerd/releases/download/v${ containerd_version }/containerd-${ containerd_version }-linux-${ image_arch }.tar.gz";
-      sha256 = "3a1322c18ee5ff4b9bd5af6b7b30c923a3eab8af1df05554f530ef8e2b24ac5e";
+      sha256 = "d66161d54546fad502fd50a13fcb79efff033fcd895adc9c44762680dcde4e69";
     };
     runc = fetchurl rec {
       passthru.url = url;
       url = "https://github.com/opencontainers/runc/releases/download/${ runc_version }/runc.${ image_arch }";
-      sha256 = "db772be63147a4e747b4fe286c7c16a2edc4a8458bd3092ea46aaee77750e8ce";
+      sha256 = "81f73a59be3d122ab484d7dfe9ddc81030f595cc59968f61c113a9a38a2c113a";
     };
     calico_crds = fetchurl rec {
       passthru.url = url;
       url = "https://github.com/projectcalico/calico/archive/${ calico_version }.tar.gz";
-      sha256 = "d25f5c9a3adeba63219f3c8425a8475ebfbca485376a78193ec1e4c74e7a6115";
+      sha256 = "481e52de684c049f3f7f7bac78f0f6f4ae424d643451adc9e3d3fa9d03fb6d57";
     };
     nerdctl = fetchurl rec {
       passthru.url = url;
       url = "https://github.com/containerd/nerdctl/releases/download/v${ nerdctl_version }/nerdctl-${ nerdctl_version }-linux-${ image_arch }.tar.gz";
-      sha256 = "ad40ecf11c689fad594a05a40fef65adb4df8ecd1ffb6711e13cff5382aeaed9";
+      sha256 = "5fc0a6e8c3a71cbba95fbdb6833fb8a7cd8e78f53de10988362d4029c14b905a";
     };
     calicoctl = fetchurl rec {
       passthru.url = url;
       url = "https://github.com/projectcalico/calico/releases/download/${ calico_version }/calicoctl-linux-${ image_arch }";
-      sha256 = "d9c04ab15bad9d8037192abd2aa4733a01b0b64a461c7b788118a0d6747c1737";
+      sha256 = "9960357ef6d61eda7abf80bd397544c1952f89d61e5eaf9f6540dae379a3ef61";
     };
     etcd = fetchurl rec {
       passthru.url = url;
       url = "https://github.com/coreos/etcd/releases/download/${ etcd_version }/etcd-${ etcd_version }-linux-${ image_arch }.tar.gz";
-      sha256 = "e13e119ff9b28234561738cd261c2a031eb1c8688079dcf96d8035b3ad19ca58";
+      sha256 = "d59017044eb776597eca480432081c5bb26f318ad292967029af1f62b588b042";
     };
     cni = fetchurl rec {
       passthru.url = url;
       url = "https://github.com/containernetworking/plugins/releases/download/${ cni_version }/cni-plugins-linux-${ image_arch }-${ cni_version }.tgz";
-      sha256 = "b275772da4026d2161bf8a8b41ed4786754c8a93ebfb6564006d5da7f23831e5";
+      sha256 = "754a71ed60a4bd08726c3af705a7d55ee3df03122b12e389fdba4bea35d7dd7e";
     };
     cassandra = fetchurl rec {
       passthru.url = url;

diff --git a/nix/scripts/mirror-apt-jammy.sh b/nix/scripts/mirror-apt-jammy.sh
@@ -26,6 +26,7 @@ shift
 # NOTE:  These are all the packages needed for all our playbooks to succeed. This list was created by trial and error
 packages=(
   python3-apt
+  python3-netaddr
   aufs-tools
   apt-transport-https
   software-properties-common

diff --git a/nix/sources.json b/nix/sources.json
@@ -17,10 +17,10 @@
         "homepage": "https://github.com/NixOS/nixpkgs",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "2c6ad38a20c99c72a54d2e471a6a5690d039687f",
-        "sha256": "1d5m7i1dpffi6s4zbkv5l04ph0xqcqg8hz0gmpd5pgik36ibk66h",
+        "rev": "057f9aecfb71c4437d2b27d3323df7f93c010b7e",
+        "sha256": "1ndiv385w1qyb3b18vw13991fzb9wg4cl21wglk89grsfsnra41k",
         "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs/archive/2c6ad38a20c99c72a54d2e471a6a5690d039687f.tar.gz",
+        "url": "https://github.com/NixOS/nixpkgs/archive/057f9aecfb71c4437d2b27d3323df7f93c010b7e.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     }
 }
diff --git a/offline/cd.sh b/offline/cd.sh
@@ -21,6 +21,3 @@ ssh -oStrictHostKeyChecking=accept-new -oConnectionAttempts=10 "root@$adminhost"
 
 # NOTE: Agent is forwarded; so that the adminhost can provision the other boxes
 ssh -A "root@$adminhost" ./bin/offline-deploy.sh
-
-
-
diff --git a/offline/ci.sh b/offline/ci.sh
@@ -32,32 +32,47 @@ rm -r binaries
 
 function list-system-containers() {
 # These are manually updated with values from
-# https://github.com/kubernetes-sigs/kubespray/blob/release-2.15/roles/download/defaults/main.yml
+# https://github.com/kubernetes-sigs/kubespray/blob/release-2.24/roles/kubespray-defaults/defaults/main/download.yml
 # TODO: Automate this. This is very wieldy :)
   cat <<EOF
-registry.k8s.io/kube-apiserver:v1.23.7
-registry.k8s.io/kube-controller-manager:v1.23.7
-registry.k8s.io/kube-scheduler:v1.23.7
-registry.k8s.io/kube-proxy:v1.23.7
-quay.io/coreos/etcd:v3.5.3
-quay.io/calico/node:v3.23.3
-quay.io/calico/cni:v3.23.3
-quay.io/calico/kube-controllers:v3.23.3
-quay.io/calico/pod2daemon-flexvol:v3.23.3
-quay.io/calico/typha:v3.23.3
-quay.io/calico/apiserver:v3.23.3
-quay.io/jetstack/cert-manager-webhook:v1.9.1
-quay.io/jetstack/cert-manager-controller:v1.9.1
-quay.io/jetstack/cert-manager-cainjector:v1.9.1
-quay.io/jetstack/cert-manager-ctl:v1.9.1
-docker.io/library/nginx:1.23.0-alpine
-registry.k8s.io/ingress-nginx/controller:v1.2.1
-registry.k8s.io/coredns:1.7.0
-registry.k8s.io/coredns/coredns:v1.8.6
-registry.k8s.io/dns/k8s-dns-node-cache:1.21.1
-registry.k8s.io/cpa/cluster-proportional-autoscaler-amd64:1.8.5
-registry.k8s.io/pause:3.6
-docker.io/kubernetesui/dashboard-amd64:v2.6.1
+registry.k8s.io/pause:3.9
+registry.k8s.io/coredns/coredns:v1.10.1
+registry.k8s.io/dns/k8s-dns-node-cache:1.22.28
+registry.k8s.io/cpa/cluster-proportional-autoscaler:v1.8.8
+registry.k8s.io/metrics-server/metrics-server:v0.6.4
+registry.k8s.io/sig-storage/local-volume-provisioner:v2.5.0
+registry.k8s.io/ingress-nginx/controller:v1.9.4
+registry.k8s.io/sig-storage/csi-attacher:v3.3.0
+registry.k8s.io/sig-storage/csi-provisioner:v3.0.0
+registry.k8s.io/sig-storage/csi-snapshotter:v5.0.0
+registry.k8s.io/sig-storage/snapshot-controller:v4.2.1
+registry.k8s.io/sig-storage/csi-resizer:v1.3.0
+registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.4.0
+registry.k8s.io/kube-apiserver:v1.28.2
+registry.k8s.io/kube-controller-manager:v1.28.2
+registry.k8s.io/kube-scheduler:v1.28.2
+registry.k8s.io/kube-proxy:v1.28.2
+quay.io/coreos/etcd:v3.5.9
+quay.io/cilium/cilium:v1.13.4
+quay.io/cilium/operator:v1.13.4
+quay.io/cilium/hubble-relay:v1.13.4
+quay.io/cilium/certgen:v0.1.8
+quay.io/cilium/hubble-ui:v0.11.0
+quay.io/cilium/hubble-ui-backend:v0.11.0
+quay.io/calico/node:v3.26.4
+quay.io/calico/cni:v3.26.4
+quay.io/calico/pod2daemon-flexvol:v3.26.4
+quay.io/calico/kube-controllers:v3.26.4
+quay.io/calico/typha:v3.26.4
+quay.io/calico/apiserver:v3.26.4
+quay.io/jetstack/cert-manager-controller:v1.13.2
+quay.io/jetstack/cert-manager-cainjector:v1.13.2
+quay.io/jetstack/cert-manager-webhook:v1.13.2
+quay.io/jetstack/cert-manager-ctl:v1.13.2
+quay.io/metallb/speaker:v0.13.9
+quay.io/metallb/controller:v0.13.9
+docker.io/library/nginx:1.25.2-alpine
+docker.io/kubernetesui/dashboard:v2.7.0
 docker.io/kubernetesui/metrics-scraper:v1.0.8
 EOF
 }
@@ -104,7 +119,7 @@ calling_charts=(
 wire_version="4.40.0"
 
 # same as prior.. in most cases.
-wire_calling_version="4.39.0"
+wire_calling_version="4.40.0"
 
 # TODO: Awaiting some fixes in wire-server regarding tagless images
 HELM_HOME=$(mktemp -d)

diff --git a/offline/docs_ubuntu_22.04.md b/offline/docs_ubuntu_22.04.md
@@ -90,14 +90,14 @@ E.g.:
 
 ```
 $ d ansible --version
-ansible [core 2.11.6] 
+ansible [core 2.15.5]
   config file = /wire-server-deploy/ansible/ansible.cfg
   configured module search path = ['/root/.ansible/plugins/modules', '/usr/share/ansible/plugins/modules']
-  ansible python module location = /nix/store/yqrs358szd85iapw6xpsh1q852f5r8wd-python3.9-ansible-core-2.11.6/lib/python3.9/site-packages/ansible
+  ansible python module location = /nix/store/p9kbf1v35r184hwx9p4snny1clkbrvp7-python3.11-ansible-core-2.15.5/lib/python3.11/site-packages/ansible
   ansible collection location = /root/.ansible/collections:/usr/share/ansible/collections
-  executable location = /nix/store/yqrs358szd85iapw6xpsh1q852f5r8wd-python3.9-ansible-core-2.11.6/bin/ansible
-  python version = 3.9.10 (main, Jan 13 2022, 23:32:03) [GCC 10.3.0]
-  jinja version = 3.0.3
+  executable location = /nix/store/p9kbf1v35r184hwx9p4snny1clkbrvp7-python3.11-ansible-core-2.15.5/bin/ansible
+  python version = 3.11.6 (main, Oct  2 2023, 13:45:54) [GCC 12.3.0] (/nix/store/qp5zys77biz7imbk6yy85q5pdv7qk84j-python3-3.11.6/bin/python3.11)
+  jinja version = 3.1.2
   libyaml = True
 
 
@@ -791,8 +791,8 @@ d kubectl cordon kubenode1
 
 first, download cert manager, and place it in the appropriate location:
 ```
-wget https://charts.jetstack.io/charts/cert-manager-v1.9.1.tgz
-tar -C ./charts -xvzf cert-manager-v1.9.1.tgz
+wget https://charts.jetstack.io/charts/cert-manager-v1.13.2.tgz
+tar -C ./charts -xvzf cert-manager-v1.13.2.tgz
 ```
 
 In case `values.yaml` and `secrets.yaml` doesn't exist yet in `./values/nginx-ingress-services` create them from templates

diff --git a/values/rabbitmq/prod-values.example.yaml b/values/rabbitmq/prod-values.example.yaml
@@ -1,5 +1,7 @@
 # More settings can be found here: https://github.com/bitnami/charts/tree/main/bitnami/rabbitmq
 rabbitmq:
+  # some Kernel versions does not support modifying ulimit via containers, setting this to empty won't override default ulimit
+  ulimitNofiles: ""
   persistence:
     size: 10Gi
     enabled: false
Original file line number	Diff line number	Diff line change
Expand Up		@@ -21,6 +21,3 @@ ssh -oStrictHostKeyChecking=accept-new -oConnectionAttempts=10 "root@$adminhost"

		# NOTE: Agent is forwarded; so that the adminhost can provision the other boxes
		ssh -A "root@$adminhost" ./bin/offline-deploy.sh