diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index f19b9af..2f4ad3e 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -5,143 +5,39 @@ env: DOCKER_BUILDKIT: 1 KUBECONFIG: ./kubeconfig jobs: + # This uses the reusable-single-node.yaml template single-node: - name: "Single node" - runs-on: ubuntu-24.04 - timeout-minutes: 40 - strategy: - fail-fast: false - matrix: - engine: [docker, nerdctl, podman] - env: - CONTAINER_ENGINE: "${{ matrix.engine }}" - steps: - - uses: actions/checkout@v4 - - name: Set up cgroup v2 delegation - run: | - sudo mkdir -p /etc/systemd/system/user@.service.d - cat </etc/hosts.u7s" >>join-command echo "cat /etc/hosts.u7s >>/etc/hosts" >>join-command $(NODE_SHELL) kubeadm token create --print-join-command | tr -d '\r' >>join-command - chmod +x join-command @echo "# Copy the 'join-command' file to another host, and run the following commands:" @echo "# On the other host (the new worker):" @echo "# make kubeadm-join" @@ -140,8 +149,7 @@ sync-external-ip: .PHONY: kubeadm-join kubeadm-join: - $(NODE_SHELL) sh -euc "envsubst /tmp/kubeadm-config.yaml" - $(NODE_SHELL) /usernetes/join-command + $(NODE_SHELL) /bin/bash /usernetes/join-command @echo "# Run 'make sync-external-ip' on the control plane" .PHONY: kubeadm-reset @@ -150,4 +158,4 @@ kubeadm-reset: .PHONY: install-flannel install-flannel: - $(NODE_SHELL) kubectl apply -f https://github.com/flannel-io/flannel/releases/download/v0.26.1/kube-flannel.yml + $(NODE_SHELL) /usernetes/Makefile.d/install-flannel.sh diff --git a/Makefile.d/install-flannel.sh b/Makefile.d/install-flannel.sh new file mode 100755 index 0000000..8f19ae1 --- /dev/null +++ b/Makefile.d/install-flannel.sh @@ -0,0 +1,89 @@ +#!/bin/bash +set -eu -o pipefail + +function INFO() { + echo >&2 -e "\e[104m\e[97m[INFO]\e[49m\e[39m $@" +} +function WARNING() { + echo >&2 -e "\e[101m\e[97m[WARNING]\e[49m\e[39m $@" +} + +function ERROR() { + echo >&2 -e "\e[101m\e[97m[ERROR]\e[49m\e[39m $@" +} + +# See chart values, 0 indicates default for platform +# https://github.com/flannel-io/flannel/blob/v0.26.1/chart/kube-flannel/values.yaml +: "${U7S_PORT_FLANNEL:='0'}" +: "${U7S_PORT_ETCD:='2379'}" + +INFO "Flannel port: ${U7S_PORT_FLANNEL}" +INFO "ETCD port: ${U7S_PORT_ETCD}" + +# Check hard dependency commands +for cmd in helm kubectl git; do + if ! command -v "${cmd}" >/dev/null 2>&1; then + ERROR "Command \"${cmd}\" is not installed" + exit 1 + fi +done + +# We need to customize the values.yaml to expose the backendPort and args +flannel_root=$(mktemp -d -u -t flannel-XXXXXXX) +git clone --quiet --depth 1 --branch v0.26.1 https://github.com/flannel-io/flannel $flannel_root +cd $flannel_root/chart + +# Write a new values.yaml that exposes what we need +cat < ./new-values.yaml +--- +global: + imagePullSecrets: +# - name: "a-secret-name" + +# The IPv4 cidr pool to create on startup if none exists. Pod IPs will be +# chosen from this range. +podCidr: "10.244.0.0/16" +podCidrv6: "" +flannel: + # kube-flannel image + image: + repository: docker.io/flannel/flannel + tag: v0.26.1 + image_cni: + repository: docker.io/flannel/flannel-cni-plugin + tag: v1.5.1-flannel2 + # flannel command arguments + enableNFTables: false + args: + - "--ip-masq" + - "--kube-subnet-mgr" + - "--etcd-endpoints=\"http://127.0.0.1:4001,https://${U7S_HOST_IP}:${U7S_PORT_ETCD},http://127.0.0.1:${U7S_PORT_ETCD}\"" +# Backend for kube-flannel. Backend should not be changed + # at runtime. (vxlan, host-gw, wireguard, udp) + # Documentation at https://github.com/flannel-io/flannel/blob/master/Documentation/backends.md + backend: "vxlan" + # Port used by the backend 0 means default value (VXLAN: 8472, Wireguard: 51821, UDP: 8285) + backendPort: ${U7S_PORT_FLANNEL} + tolerations: + - effect: NoExecute + operator: Exists + - effect: NoSchedule + operator: Exists + +netpol: + enabled: false +EOF + +mv ./new-values.yaml ./kube-flannel/values.yaml + +# Run this first in case a failure with kubectl +kubectl get pods -n kube-flannel +# Fall back to warning so a re-install does not fail +kubectl create namespace kube-flannel || WARNING "kube-flannel namespace is already created" +kubectl label --overwrite namespace kube-flannel pod-security.kubernetes.io/enforce=privileged || true +# If the command is issued again, this cleanup is needed +helm delete flannel --namespace kube-flannel kube-flannel || true +# We could also do --set flannel.backendPort= but it's the same to set as the default +helm install flannel --namespace kube-flannel kube-flannel +cd - +rm -rf $flannel_root diff --git a/docker-compose.yaml b/docker-compose.yaml index 15d8e9f..3e2161a 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -11,14 +11,15 @@ services: default: ipv4_address: ${U7S_NODE_IP} ports: - # etcd - - 2379:2379 - # kube-apiserver - - 6443:6443 - # kubelet - - 10250:10250 - # flannel - - 8472:8472/udp + # : + # etcd (default: 2379) + - ${U7S_PORT_ETCD}:${U7S_PORT_ETCD} + # kube-apiserver (default: 6443) + - ${U7S_PORT_KUBE_APISERVER}:${U7S_PORT_KUBE_APISERVER} + # kubelet (default: 10250) + - ${U7S_PORT_KUBELET}:${U7S_PORT_KUBELET} + # flannel (default: 8472) + - ${U7S_PORT_FLANNEL}:${U7S_PORT_FLANNEL}/udp volumes: - .:/usernetes:ro - /boot:/boot:ro diff --git a/hack/create-cluster-lima.sh b/hack/create-cluster-lima.sh index 008c392..1a86c95 100755 --- a/hack/create-cluster-lima.sh +++ b/hack/create-cluster-lima.sh @@ -5,6 +5,10 @@ set -eux -o pipefail : "${LIMA_TEMPLATE:=template://default}" : "${CONTAINER_ENGINE:=docker}" : "${LOCKDOWN_SUDO:=1}" +: "${U7S_PORT_KUBE_APISERVER:=6443}" +: "${U7S_PORT_ETCD:=2379}" +: "${U7S_PORT_FLANNEL:=8472}" +: "${U7S_PORT_KUBELET:=10250}" guest_home="/home/${USER}.linux" @@ -24,21 +28,23 @@ for host in host0 host1; do ${LIMACTL} shell "${host}" CONTAINER_ENGINE="${CONTAINER_ENGINE}" "${guest_home}/usernetes/init-host/init-host.rootless.sh" done +U7S_SERVICE_PORTS="U7S_PORT_KUBE_APISERVER=${U7S_PORT_KUBE_APISERVER} U7S_PORT_ETCD=${U7S_PORT_ETCD} U7S_PORT_FLANNEL=${U7S_PORT_FLANNEL} U7S_PORT_KUBELET=${U7S_PORT_KUBELET}" + # Launch a Kubernetes node inside a Rootless Docker host for host in host0 host1; do - ${LIMACTL} shell "${host}" CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C "${guest_home}/usernetes" up + ${LIMACTL} shell "${host}" ${U7S_SERVICE_PORTS} CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C "${guest_home}/usernetes" up done # Bootstrap a cluster with host0 -${LIMACTL} shell host0 CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C "${guest_home}/usernetes" kubeadm-init install-flannel kubeconfig join-command +${LIMACTL} shell host0 ${U7S_SERVICE_PORTS} CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C "${guest_home}/usernetes" kubeadm-init install-flannel kubeconfig join-command # Let host1 join the cluster ${LIMACTL} copy host0:~/usernetes/join-command host1:~/usernetes/join-command -${LIMACTL} shell host1 CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C "${guest_home}/usernetes" kubeadm-join -${LIMACTL} shell host0 CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C "${guest_home}/usernetes" sync-external-ip +${LIMACTL} shell host1 ${U7S_SERVICE_PORTS} CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C "${guest_home}/usernetes" kubeadm-join +${LIMACTL} shell host0 ${U7S_SERVICE_PORTS} CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C "${guest_home}/usernetes" sync-external-ip # Enable kubectl -ssh -q -f -N -L 6443:127.0.0.1:6443 -F ~/.lima/host0/ssh.config lima-host0 +ssh -q -f -N -L ${U7S_PORT_KUBE_APISERVER}:127.0.0.1:${U7S_PORT_KUBE_APISERVER} -F ~/.lima/host0/ssh.config lima-host0 ${LIMACTL} copy host0:${guest_home}/usernetes/kubeconfig ./kubeconfig KUBECONFIG="$(pwd)/kubeconfig" export KUBECONFIG diff --git a/kubeadm-config.yaml b/kubeadm-config.yaml index cc09f9a..2b40ad8 100644 --- a/kubeadm-config.yaml +++ b/kubeadm-config.yaml @@ -1,31 +1,48 @@ ---- -apiVersion: kubeadm.k8s.io/v1beta3 +apiVersion: kubeadm.k8s.io/v1beta4 kind: InitConfiguration +localAPIEndpoint: + bindPort: ${U7S_PORT_KUBE_APISERVER} --- -apiVersion: kubeadm.k8s.io/v1beta3 +apiVersion: kubeadm.k8s.io/v1beta4 kind: ClusterConfiguration -networking: - serviceSubnet: "10.96.0.0/16" - podSubnet: "10.244.0.0/16" -controlPlaneEndpoint: "${U7S_NODE_NAME}:6443" +controlPlaneEndpoint: "${U7S_NODE_NAME}:${U7S_PORT_KUBE_APISERVER}" apiServer: - extraArgs: - advertise-address: "${U7S_HOST_IP}" - cloud-provider: external - # Default: "Hostname,InternalDNS,InternalIP,ExternalDNS,ExternalIP" - kubelet-preferred-address-types: "ExternalIP" certSANs: - - localhost - - 127.0.0.1 - - "${U7S_NODE_NAME}" - - "${U7S_HOST_IP}" + - localhost + - 127.0.0.1 + - "${U7S_NODE_NAME}" + - "${U7S_HOST_IP}" + extraArgs: + - name: etcd-servers + value: https://127.0.0.1:${U7S_PORT_ETCD} + - name: advertise-address + value: ${U7S_HOST_IP} + - name: secure-port + value: "${U7S_PORT_KUBE_APISERVER}" + - name: cloud-provider + value: external + # Default: "Hostname,InternalDNS,InternalIP,ExternalDNS,ExternalIP" + - name: kubelet-preferred-address-types + value: ExternalIP controllerManager: extraArgs: - cloud-provider: external + - name: cloud-provider + value: external +networking: + serviceSubnet: "10.96.0.0/16" + podSubnet: "10.244.0.0/16" +etcd: + local: + extraArgs: + - name: listen-client-urls + value: "https://127.0.0.1:${U7S_PORT_ETCD},https://${U7S_NODE_IP}:${U7S_PORT_ETCD}" + - name: advertise-client-urls + value: https://${U7S_NODE_IP}:${U7S_PORT_ETCD} --- kind: KubeletConfiguration apiVersion: kubelet.config.k8s.io/v1beta1 failSwapOn: false +port: ${U7S_PORT_KUBELET} featureGates: KubeletInUserNamespace: true ---