[Release-1.29] - Add nft to k3s-root userspace bundle

[dereknola]

Backport fix for Add nft to k3s-root userspace bundle

• Add nft to k3s-root userspace bundle #11267

[fmoral2]

Validated on Version:

-$ k3s version v1.29.10+k3s-c50a9962 (c50a9962)

Environment Details

Infrastructure
Cloud EC2 instance

Node(s) CPU architecture, OS, and Version:
ubuntu
AMD

Cluster Configuration:
-1 node server

Steps to validate the fix

1. Install k3s with arg for kube-proxy-arg=proxy-mode=nftables
2. Make sure that nft is not installed or present on the host
3. Validate k3s starts and works just fine as expected.

Reproduction Issue:

~$ which nft
 ~$  EMPTY


 ~$ curl -fL https://get.k3s.io | INSTALL_K3S_COMMIT=5159131543ae120d69c2e42b746e7f73cb917a59   sh -s - server  --kube-proxy-arg=proxy-mode=nftables --write-kubeconfig-mode 644
 

~$ sudo systemctl cat k3s.service
  

[Service]
Type=notify
EnvironmentFile=-/etc/default/%N
EnvironmentFile=-/etc/sysconfig/%N
EnvironmentFile=-/etc/systemd/system/k3s.service.env
KillMode=process
Delegate=yes
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=1048576
LimitNPROC=infinity
LimitCORE=infinity
TasksMax=infinity
TimeoutStartSec=0
Restart=always
RestartSec=5s
ExecStartPre=/bin/sh -xc '! /usr/bin/systemctl is-enabled --quiet nm-cloud-setup.service 2>/dev/null'
ExecStartPre=-/sbin/modprobe br_netfilter
ExecStartPre=-/sbin/modprobe overlay
ExecStart=/usr/local/bin/k3s \
    server \
        '--kube-proxy-arg=proxy-mode=nftables' \
        '--write-k



$ sudo systemctl status k3s
● k3s.service - Lightweight Kubernetes
     Loaded: loaded (/etc/systemd/system/k3s.service; enabled; vendor preset: enabled)
     Active: activating (auto-restart) (Result: exit-code) since Wed 2024-11-13 19:05:39 UTC; 772ms ago
       Docs: https://k3s.io
    Process: 34310 ExecStartPre=/bin/sh -xc ! /usr/bin/systemctl is-enabled --quiet nm-cloud-setup.service 2>/dev/null (code=exited, status=0/SUCCESS)
    Process: 34312 ExecStartPre=/sbin/modprobe br_netfilter (code=exited, status=0/SUCCESS)
    Process: 34313 ExecStartPre=/sbin/modprobe overlay (code=exited, status=0/SUCCESS)
    Process: 34314 ExecStart=/usr/local/bin/k3s server --kube-proxy-arg=proxy-mode=nftables --write-kubeconfig-mode 644 (code=exited, status=1/FAILURE)
   Main PID: 34314 (code=exited, status=1/FAILURE)
        CPU: 11.339s



$ journalctl -xeu k3s.service | grep "unable to create proxier"
Nov 13 20:39:59 ip- k3s[149509]: E1113 20:39:59.177715  149509 server.go:127] "Error running ProxyServer" err="unable to create proxier: unable to create ipv4 proxier: could not find nftables binary: exec: \"nft\": executable file not found in $PATH"
Nov 13 20:39:59 ip- k3s[149509]: Error: unable to create proxier: unable to create ipv4 proxier: could not find nftables binary: exec: "nft": executable file not found in $PATH
Nov 13 20:39:59 ip- k3s[149509]: time="2024-11-13T20:39:59Z" level=error msg="kube-proxy exited: unable to create proxier: unable to create ipv4 proxier: could not find nftables binary: exec: \"nft\": executable file not found in $PATH"
Nov 13 20:40:12 ip- k3s[149821]: E1113 20:40:12.634044  149821 server.go:127] "Error running ProxyServer" err="unable to create proxier: unable to create ipv4 proxier: could not find nftables binary: exec: \"nft\": executable file not found in $PATH"
Nov 13 20:40:12 ip- k3s[149821]: Error: unable to create proxier: unable to create ipv4 proxier: could not find nftables binary: exec: "nft": executable file not found in $PATH
Nov 13 20:40:12 ip- k3s[149821]: time="2024-11-13T20:40:12Z" level=error msg="kube-proxy exited: unable to create proxier: unable to create ipv4 proxier: could not find nftables binary: exec: \"nft\": executable file not found in $PATH"




$ kubectl get pods -A
The connection to the server 127.0.0.1:6443 was refused - did you specify the right host or port?

 

Validation Results:

  
 which nft
  empty
 
curl -fL https://get.k3s.io | INSTALL_K3S_COMMIT=5d2bc57e85434c090299021a6f16dcc64c7fde09   sh -s - server  --kube-proxy-arg=proxy-mode=nftables --write-kubeconfig-mode 644

~$ sudo systemctl cat k3s.service

[Service]
Type=notify
EnvironmentFile=-/etc/default/%N
EnvironmentFile=-/etc/sysconfig/%N
EnvironmentFile=-/etc/systemd/system/k3s.service.env
KillMode=process
Delegate=yes
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=1048576
LimitNPROC=infinity
LimitCORE=infinity
TasksMax=infinity
TimeoutStartSec=0
Restart=always
RestartSec=5s
ExecStartPre=/bin/sh -xc '! /usr/bin/systemctl is-enabled --quiet nm-cloud-setup.service 2>/dev/null'
ExecStartPre=-/sbin/modprobe br_netfilter
ExecStartPre=-/sbin/modprobe overlay
ExecStart=/usr/local/bin/k3s \
    server \
        '--kube-proxy-arg=proxy-mode=nftables' \
        '--write-k


$ journalctl -xeu k3s.service | grep "unable to create proxier"
<Empty>

$ sudo systemctl status k3s
● k3s.service - Lightweight Kubernetes
     Active: active (running) since Wed 2024-11-13 21:19:18 UTC; 59s ago
       Docs: https://k3s.io
    Process: 27084 ExecStartPre=/bin/sh -xc ! /usr/bin/systemctl is-enabled --quiet nm-cloud-setup.service 2>/dev/null (code=exited, status=0/SUCCESS)
    Process: 27086 ExecStartPre=/sbin/modprobe br_netfilter (code=exited, status=0/SUCCESS)
    Process: 27087 ExecStartPre=/sbin/modprobe overlay (code=exited, status=0/SUCCESS)

    Process: 7425 ExecStartPre=/sbin/modprobe overlay (code=exited, status=0/SUCCESS)
   Main PID: 7426 (k3s-server)
      Tasks: 209
     Memory: 1.8G
        CPU: 9min 18.199s
     CGroup: /system.slice/k3s.service


   $ kubectl apply -f https://gist.githubusercontent.com/rancher-max/5b160babb714d8d5a123df6a24ec9b3d/raw/7e2d36fbf735e6d1e2a8e10cc2cf1ce19ea7c978/workloads.yaml
namespace/auto-clusterip created
deployment.apps/test-clusterip created
service/nginx-clusterip-svc created
namespace/auto-daemonset created
daemonset.apps/test-daemonset created
namespace/auto-dns created
pod/dnsutils created
namespace/auto-ingress created
ingress.networking.k8s.io/test-ingress created
networkpolicy.networking.k8s.io/allow-all-ingress created
service/nginx-ingress-svc created
replicationcontroller/test-ingress created
namespace/auto-nodeport created
deployment.apps/test-nodeport created
service/nginx-nodeport-svc created


 $ k get pods -A
NAMESPACE        NAME                                      READY   STATUS      RESTARTS      AGE
auto-clusterip   test-clusterip-6b86dc97bd-2z4lq           1/1     Running     0             92m
auto-clusterip   test-clusterip-6b86dc97bd-6rrmt           1/1     Running     0             92m
auto-daemonset   test-daemonset-qn984                      1/1     Running     0             92m
auto-dns         dnsutils                                  1/1     Running     1 (32m ago)   92m
auto-ingress     test-ingress-2sbw8                        1/1     Running     0             92m
auto-ingress     test-ingress-kqkm8                        1/1     Running     0             92m
auto-nodeport    test-nodeport-655c76c448-j4jm9            1/1     Running     0             92m
auto-nodeport    test-nodeport-655c76c448-ws9mp            1/1     Running     0             92m
kube-system      coredns-56f6fc8fd7-tkhsj                  1/1     Running     0             118m
kube-system      helm-install-traefik-crd-clbkh            0/1     Completed   0             118m
kube-system      helm-install-traefik-zmmsn                0/1     Completed   1             118m
kube-system      local-path-provisioner-5cf85fd84d-2tw5x   1/1     Running     0             118m
kube-system      metrics-server-5985cbc9d7-p7gh9           1/1     Running     0             118m
kube-system      svclb-traefik-03ba9fd8-5tzgc              2/2     Running     0             118m
kube-system      traefik-57b79cf995-jn4wp                  1/1     Running     0             118m




 $ journalctl -xeu k3s.service | grep "kube-proxy"
Nov 13 21:15:22 ip-   k3s[24076]: time="2024-11-13T21:15:22Z" level=info msg="certificate CN=system:kube-proxy signed by CN=k3s-client-ca@1731532522: notBefore=2024-11-13 21:15:22 +0000 UTC notAfter=2025-11-13 21:15:22 +0000 UTC"
Nov 13 21:15:27 ip-   k3s[24076]: time="2024-11-13T21:15:27Z" level=info msg="Running kube-proxy --cluster-cidr=10.42.0.0/16 --conntrack-max-per-core=0 --conntrack-tcp-timeout-close-wait=0s --conntrack-tcp-timeout-established=0s --feature-gates=NFTablesProxyMode=true --healthz-bind-address=127.0.0.1 --hostname-override=ip-   --kubeconfig=/var/lib/rancher/k3s/agent/kubeproxy.kubeconfig --proxy-mode=iptables"
Nov 13 21:15:27 ip-   k3s[24076]: I1113 21:15:27.224503   24076 server.go:659] "kube-proxy running in dual-stack mode" primary ipFamily="IPv4"