From a864afca8f0129a7d4de25d346555ad283a599c6 Mon Sep 17 00:00:00 2001 From: Yun Li Date: Wed, 6 Mar 2024 15:04:24 +0800 Subject: [PATCH 1/7] Add one substitute mirror source to mitigate k8s source deprecation issue --- files/image_config/kubernetes/kubernetes.list | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/files/image_config/kubernetes/kubernetes.list b/files/image_config/kubernetes/kubernetes.list index 5c888b830623..31932f81fbce 100644 --- a/files/image_config/kubernetes/kubernetes.list +++ b/files/image_config/kubernetes/kubernetes.list @@ -1,4 +1,6 @@ # The following is as recommended by https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/ # Whenever an OS update from Debian stretch is done, make sure to find the matching k8s sources list # -deb https://apt.kubernetes.io/ kubernetes-xenial main +# The apt.kubernetes.io is deprecated, need to use this mirror to mitigate this issue first +# TODO: upgrade k8s version +deb https://mirror.azure.cn/kubernetes/packages/apt/mirror/apt.kubernetes.io/ kubernetes-xenial main From d41f7b8179d5b62fb8504b0b2c9b632c4fdeed00 Mon Sep 17 00:00:00 2001 From: Yun Li Date: Wed, 6 Mar 2024 15:05:41 +0800 Subject: [PATCH 2/7] Trigger k8s packages build --- rules/config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules/config b/rules/config index 026cf34fb79f..5b2cad9bcbda 100644 --- a/rules/config +++ b/rules/config @@ -185,7 +185,7 @@ INCLUDE_ROUTER_ADVERTISER ?= y # INCLUDE_KUBERNETES - if set to y kubernetes packages are installed to be able to # run as worker node in kubernetes cluster. -INCLUDE_KUBERNETES ?= n +INCLUDE_KUBERNETES ?= y KUBE_DOCKER_PROXY = http://172.16.1.1:3128/ From 96b7bbda10d3b181f0668388a5bfbce7d15e0124 Mon Sep 17 00:00:00 2001 From: Yun Li Date: Wed, 6 Mar 2024 19:51:22 +0800 Subject: [PATCH 3/7] Install k8s packages from storage --- Makefile.work | 1 + build_debian.sh | 29 +++++++++++++------ files/image_config/kubernetes/kubernetes.list | 4 +-- rules/config | 1 + 4 files changed, 23 insertions(+), 12 deletions(-) diff --git a/Makefile.work b/Makefile.work index a507439679e9..1a7ec26664ea 100644 --- a/Makefile.work +++ b/Makefile.work @@ -530,6 +530,7 @@ SONIC_BUILD_INSTRUCTION := $(MAKE) \ INCLUDE_KUBERNETES=$(INCLUDE_KUBERNETES) \ KUBERNETES_VERSION=$(KUBERNETES_VERSION) \ KUBERNETES_CNI_VERSION=$(KUBERNETES_CNI_VERSION) \ + KUBERNETES_CRI_TOOLS_VERSION=$(KUBERNETES_CRI_TOOLS_VERSION) \ K8s_GCR_IO_PAUSE_VERSION=$(K8s_GCR_IO_PAUSE_VERSION) \ INCLUDE_KUBERNETES_MASTER=$(INCLUDE_KUBERNETES_MASTER) \ SONIC_ENABLE_PFCWD_ON_START=$(ENABLE_PFCWD_ON_START) \ diff --git a/build_debian.sh b/build_debian.sh index 207bc95a7142..a4cd07e99276 100755 --- a/build_debian.sh +++ b/build_debian.sh @@ -271,16 +271,27 @@ sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install docker-ce=${DOCKER_VERSIO install_kubernetes () { local ver="$1" + ## Install k8s package from storage + local storage_prefix="https://sonicstorage.blob.core.windows.net/public/kubernetes" sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT curl -fsSL \ - https://packages.cloud.google.com/apt/doc/apt-key.gpg | \ - sudo LANG=C chroot $FILESYSTEM_ROOT tee /etc/apt/trusted.gpg.d/kubernetes.asc - ## Check out the sources list update matches current Debian version - sudo cp files/image_config/kubernetes/kubernetes.list $FILESYSTEM_ROOT/etc/apt/sources.list.d/ - sudo LANG=C chroot $FILESYSTEM_ROOT apt-get update - sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install kubernetes-cni=${KUBERNETES_CNI_VERSION} - sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install kubelet=${ver} - sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install kubectl=${ver} - sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install kubeadm=${ver} + ${storage_prefix}/apt-key.gpg | sudo LANG=C chroot $FILESYSTEM_ROOT tee /etc/apt/trusted.gpg.d/kubernetes.asc + sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT curl -o /tmp/cri-tools.deb -fsSL \ + ${storage_prefix}/cri-tools_${KUBERNETES_CRI_TOOLS_VERSION}_${CONFIGURED_ARCH}.deb + sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT curl -o /tmp/kubernetes-cni.deb -fsSL \ + ${storage_prefix}/kubernetes-cni_${KUBERNETES_CNI_VERSION}_${CONFIGURED_ARCH}.deb + sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT curl -o /tmp/kubelet.deb -fsSL \ + ${storage_prefix}/kubelet_${ver}_${CONFIGURED_ARCH}.deb + sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT curl -o /tmp/kubeadm.deb -fsSL \ + ${storage_prefix}/kubeadm_${ver}_${CONFIGURED_ARCH}.deb + sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT curl -o /tmp/kubectl.deb -fsSL \ + ${storage_prefix}/kubectl_${ver}_${CONFIGURED_ARCH}.deb + + sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install -f /tmp/cri-tools.deb + sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install -f /tmp/kubernetes-cni.deb + sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install -f /tmp/kubelet.deb + sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install -f /tmp/kubeadm.deb + sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install -f /tmp/kubectl.deb + sudo LANG=C chroot $FILESYSTEM_ROOT rm -f /tmp/{cri-tools,kubernetes-cni,kubelet,kubeadm,kubectl}.deb } if [ "$INCLUDE_KUBERNETES" == "y" ] diff --git a/files/image_config/kubernetes/kubernetes.list b/files/image_config/kubernetes/kubernetes.list index 31932f81fbce..5c888b830623 100644 --- a/files/image_config/kubernetes/kubernetes.list +++ b/files/image_config/kubernetes/kubernetes.list @@ -1,6 +1,4 @@ # The following is as recommended by https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/ # Whenever an OS update from Debian stretch is done, make sure to find the matching k8s sources list # -# The apt.kubernetes.io is deprecated, need to use this mirror to mitigate this issue first -# TODO: upgrade k8s version -deb https://mirror.azure.cn/kubernetes/packages/apt/mirror/apt.kubernetes.io/ kubernetes-xenial main +deb https://apt.kubernetes.io/ kubernetes-xenial main diff --git a/rules/config b/rules/config index 5b2cad9bcbda..3d0efa6fd9be 100644 --- a/rules/config +++ b/rules/config @@ -196,6 +196,7 @@ KUBE_DOCKER_PROXY = http://172.16.1.1:3128/ # KUBERNETES_VERSION = 1.22.2-00 KUBERNETES_CNI_VERSION = 0.8.7-00 +KUBERNETES_CRI_TOOLS_VERSION = 1.26.0-00 K8s_GCR_IO_PAUSE_VERSION = 3.5 # INCLUDE_KUBERNETES_MASTER - if set to y kubernetes packages are installed o be able From e483cab37fb65c8554984783b9fa93bb7c1b7d76 Mon Sep 17 00:00:00 2001 From: Yun Li Date: Wed, 6 Mar 2024 19:52:44 +0800 Subject: [PATCH 4/7] Fix line format --- Makefile.work | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.work b/Makefile.work index 1a7ec26664ea..18ff536f501e 100644 --- a/Makefile.work +++ b/Makefile.work @@ -530,7 +530,7 @@ SONIC_BUILD_INSTRUCTION := $(MAKE) \ INCLUDE_KUBERNETES=$(INCLUDE_KUBERNETES) \ KUBERNETES_VERSION=$(KUBERNETES_VERSION) \ KUBERNETES_CNI_VERSION=$(KUBERNETES_CNI_VERSION) \ - KUBERNETES_CRI_TOOLS_VERSION=$(KUBERNETES_CRI_TOOLS_VERSION) \ + KUBERNETES_CRI_TOOLS_VERSION=$(KUBERNETES_CRI_TOOLS_VERSION) \ K8s_GCR_IO_PAUSE_VERSION=$(K8s_GCR_IO_PAUSE_VERSION) \ INCLUDE_KUBERNETES_MASTER=$(INCLUDE_KUBERNETES_MASTER) \ SONIC_ENABLE_PFCWD_ON_START=$(ENABLE_PFCWD_ON_START) \ From 5e48a1abb9e9ca3746753e993d0d432126149d22 Mon Sep 17 00:00:00 2001 From: Yun Li Date: Wed, 6 Mar 2024 21:19:39 +0800 Subject: [PATCH 5/7] Change package install order --- build_debian.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build_debian.sh b/build_debian.sh index a4cd07e99276..8ace65c2c421 100755 --- a/build_debian.sh +++ b/build_debian.sh @@ -281,16 +281,16 @@ install_kubernetes () { ${storage_prefix}/kubernetes-cni_${KUBERNETES_CNI_VERSION}_${CONFIGURED_ARCH}.deb sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT curl -o /tmp/kubelet.deb -fsSL \ ${storage_prefix}/kubelet_${ver}_${CONFIGURED_ARCH}.deb - sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT curl -o /tmp/kubeadm.deb -fsSL \ - ${storage_prefix}/kubeadm_${ver}_${CONFIGURED_ARCH}.deb sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT curl -o /tmp/kubectl.deb -fsSL \ ${storage_prefix}/kubectl_${ver}_${CONFIGURED_ARCH}.deb + sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT curl -o /tmp/kubeadm.deb -fsSL \ + ${storage_prefix}/kubeadm_${ver}_${CONFIGURED_ARCH}.deb sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install -f /tmp/cri-tools.deb sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install -f /tmp/kubernetes-cni.deb sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install -f /tmp/kubelet.deb - sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install -f /tmp/kubeadm.deb sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install -f /tmp/kubectl.deb + sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install -f /tmp/kubeadm.deb sudo LANG=C chroot $FILESYSTEM_ROOT rm -f /tmp/{cri-tools,kubernetes-cni,kubelet,kubeadm,kubectl}.deb } From ba593d77ea9bf53af2f25e370d68270365e2ab4c Mon Sep 17 00:00:00 2001 From: Yun Li Date: Thu, 7 Mar 2024 00:23:35 +0800 Subject: [PATCH 6/7] Trigger build master image --- build_debian.sh | 2 -- rules/config | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/build_debian.sh b/build_debian.sh index 8ace65c2c421..b9dfd47bb5a4 100755 --- a/build_debian.sh +++ b/build_debian.sh @@ -273,8 +273,6 @@ install_kubernetes () { local ver="$1" ## Install k8s package from storage local storage_prefix="https://sonicstorage.blob.core.windows.net/public/kubernetes" - sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT curl -fsSL \ - ${storage_prefix}/apt-key.gpg | sudo LANG=C chroot $FILESYSTEM_ROOT tee /etc/apt/trusted.gpg.d/kubernetes.asc sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT curl -o /tmp/cri-tools.deb -fsSL \ ${storage_prefix}/cri-tools_${KUBERNETES_CRI_TOOLS_VERSION}_${CONFIGURED_ARCH}.deb sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT curl -o /tmp/kubernetes-cni.deb -fsSL \ diff --git a/rules/config b/rules/config index 3d0efa6fd9be..63f9e25ce6c6 100644 --- a/rules/config +++ b/rules/config @@ -185,7 +185,7 @@ INCLUDE_ROUTER_ADVERTISER ?= y # INCLUDE_KUBERNETES - if set to y kubernetes packages are installed to be able to # run as worker node in kubernetes cluster. -INCLUDE_KUBERNETES ?= y +INCLUDE_KUBERNETES ?= n KUBE_DOCKER_PROXY = http://172.16.1.1:3128/ @@ -201,7 +201,7 @@ K8s_GCR_IO_PAUSE_VERSION = 3.5 # INCLUDE_KUBERNETES_MASTER - if set to y kubernetes packages are installed o be able # to run as master node in kubernetes cluster -INCLUDE_KUBERNETES_MASTER ?= n +INCLUDE_KUBERNETES_MASTER ?= y # MASTER_KUBERNETES_VERSION - version of k8s components # MASTER_PAUSE_VERSION - version of pause container image From c7594e78ca32cabdb18347432890e018395f6f96 Mon Sep 17 00:00:00 2001 From: Yun Li Date: Thu, 7 Mar 2024 04:41:39 +0800 Subject: [PATCH 7/7] Disable k8s master image build --- rules/config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules/config b/rules/config index 63f9e25ce6c6..4628313dc179 100644 --- a/rules/config +++ b/rules/config @@ -201,7 +201,7 @@ K8s_GCR_IO_PAUSE_VERSION = 3.5 # INCLUDE_KUBERNETES_MASTER - if set to y kubernetes packages are installed o be able # to run as master node in kubernetes cluster -INCLUDE_KUBERNETES_MASTER ?= y +INCLUDE_KUBERNETES_MASTER ?= n # MASTER_KUBERNETES_VERSION - version of k8s components # MASTER_PAUSE_VERSION - version of pause container image