[release-1.23] Backports for 2022-12

cmd/k3s/main.go

@@ -3,9 +3,11 @@ package main
 import (
 	"bytes"
 	"context"
+	"io"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"syscall"
 
@@ -20,14 +22,15 @@ import (
 	"github.com/pkg/errors"
 	"github.com/rancher/wrangler/pkg/resolvehome"
 	"github.com/sirupsen/logrus"
+	"github.com/spf13/pflag"
 	"github.com/urfave/cli"
 )
 
 var criDefaultConfigPath = "/etc/crictl.yaml"
 
 // main entrypoint for the k3s multicall binary
 func main() {
-	dataDir := findDataDir()
+	dataDir := findDataDir(os.Args)
 
 	// Handle direct invocation via symlink alias (multicall binary behavior)
 	if runCLIs(dataDir) {
@@ -79,19 +82,17 @@ func main() {
 // findDataDir reads data-dir settings from the CLI args and config file.
 // If not found, the default will be used, which varies depending on whether
 // k3s is being run as root or not.
-func findDataDir() string {
-	for i, arg := range os.Args {
-		for _, flagName := range []string{"--data-dir", "-d"} {
-			if flagName == arg {
-				if len(os.Args) > i+1 {
-					return os.Args[i+1]
-				}
-			} else if strings.HasPrefix(arg, flagName+"=") {
-				return arg[len(flagName)+1:]
-			}
-		}
+func findDataDir(args []string) string {
+	var dataDir string
+	fs := pflag.NewFlagSet("data-dir-set", pflag.ContinueOnError)
+	fs.ParseErrorsWhitelist.UnknownFlags = true
+	fs.SetOutput(io.Discard)
+	fs.StringVarP(&dataDir, "data-dir", "d", "", "Data directory")
+	fs.Parse(args)
+	if dataDir != "" {
+		return dataDir
 	}
-	dataDir := configfilearg.MustFindString(os.Args, "data-dir")
+	dataDir = configfilearg.MustFindString(args, "data-dir")
 	if d, err := datadir.Resolve(dataDir); err == nil {
 		dataDir = d
 	} else {
@@ -100,6 +101,24 @@ func findDataDir() string {
 	return dataDir
 }
 
+// findPreferBundledBin searches for prefer-bundled-bin from the config file, then CLI args.
+// we use pflag to process the args because we not yet parsed flags bound to the cli.Context
+func findPreferBundledBin(args []string) bool {
+	var preferBundledBin bool
+	fs := pflag.NewFlagSet("prefer-set", pflag.ContinueOnError)
+	fs.ParseErrorsWhitelist.UnknownFlags = true
+	fs.SetOutput(io.Discard)
+	fs.BoolVar(&preferBundledBin, "prefer-bundled-bin", false, "Prefer bundled binaries")
+
+	preferRes := configfilearg.MustFindString(args, "prefer-bundled-bin")
+	if preferRes != "" {
+		preferBundledBin, _ = strconv.ParseBool(preferRes)
+	}
+
+	fs.Parse(args)
+	return preferBundledBin
+}
+
 // runCLIs handles the case where the binary is being executed as a symlink alias,
 // /usr/local/bin/crictl for example. If the executable name is one of the external
 // binaries, it calls it directly and returns true. If it's not an external binary,
@@ -158,7 +177,13 @@ func stageAndRun(dataDir, cmd string, args []string) error {
 	}
 	logrus.Debugf("Asset dir %s", dir)
 
-	if err := os.Setenv("PATH", filepath.Join(dir, "bin")+":"+os.Getenv("PATH")+":"+filepath.Join(dir, "bin/aux")); err != nil {
+	var pathEnv string
+	if findPreferBundledBin(args) {
+		pathEnv = filepath.Join(dir, "bin") + ":" + filepath.Join(dir, "bin/aux") + ":" + os.Getenv("PATH")
+	} else {
+		pathEnv = filepath.Join(dir, "bin") + ":" + os.Getenv("PATH") + ":" + filepath.Join(dir, "bin/aux")
+	}
+	if err := os.Setenv("PATH", pathEnv); err != nil {
 		return err
 	}
 	if err := os.Setenv(version.ProgramUpper+"_DATA_DIR", dir); err != nil {

cmd/k3s/main_test.go

@@ -0,0 +1,59 @@
+package main
+
+import "testing"
+
+func Test_UnitFindPreferBundledBin(t *testing.T) {
+	tests := []struct {
+		name string
+		args []string
+		want bool
+	}{
+		{
+			name: "Single argument",
+			args: []string{"--prefer-bundled-bin"},
+			want: true,
+		},
+		{
+			name: "no argument",
+			args: []string{""},
+			want: false,
+		},
+		{
+			name: "Argument with equal true",
+			args: []string{"--prefer-bundled-bin=true"},
+			want: true,
+		},
+		{
+			name: "Argument with equal false",
+			args: []string{"--prefer-bundled-bin=false"},
+			want: false,
+		},
+		{
+			name: "Argument with equal 1",
+			args: []string{"--prefer-bundled-bin=1"},
+			want: true,
+		},
+		{
+			name: "Argument with equal 0",
+			args: []string{"--prefer-bundled-bin=0"},
+			want: false,
+		},
+		{
+			name: "Multiple arguments",
+			args: []string{"--abcd", "--prefer-bundled-bin", "--efgh"},
+			want: true,
+		},
+		{
+			name: "Repeated arguments",
+			args: []string{"--abcd", "--prefer-bundled-bin=false", "--prefer-bundled-bin"},
+			want: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := findPreferBundledBin(tt.args); got != tt.want {
+				t.Errorf("findPreferBundledBin() = %+v\nWant = %+v", got, tt.want)
+			}
+		})
+	}
+}

docs/adrs/k3s-charts.md

@@ -0,0 +1,23 @@
+# Stage Helm charts through k3s-charts
+
+Date: 2022-11-17
+
+## Status
+
+Accepted
+
+## Context
+
+The upstream Traefik chart repo has seen significant changes over the last month. Upstream has changed their repo structure, and actively removed content from deprecated locations,
+at least twice. In both cases, this immediately broke K3s CI, requiring changes to our build scripts in order to restore the ability to build, test, and package K3s.
+
+The K3s chart build process also makes several changes to the upstream chart to add values and break out the CRDs, using an ad-hoc set of scripts that are difficult to maintain.
+There are better tools available to perform this same task, if we did so in a dedicated repo.
+
+## Decision
+
+We will make use of the [charts-build-scripts](https://github.com/rancher/charts-build-scripts) tool to customize the upstream chart and stage it through a stable intermediate repo.
+
+## Consequences
+
+When updating Helm charts distributed with K3s, additional pull requests will be necessary to stage new versions into the k3s-io/k3s-charts repo, before updating the chart version in K3s.

go.mod

@@ -90,7 +90,7 @@ require (
 	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/websocket v1.4.2
 	github.com/gruntwork-io/terratest v0.40.6
-	github.com/k3s-io/helm-controller v0.13.0
+	github.com/k3s-io/helm-controller v0.13.1
 	github.com/k3s-io/kine v0.9.6
 	github.com/klauspost/compress v1.15.9
 	github.com/kubernetes-sigs/cri-tools v0.0.0-00010101000000-000000000000
@@ -112,6 +112,7 @@ require (
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/rootless-containers/rootlesskit v1.0.1
 	github.com/sirupsen/logrus v1.9.0
+	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.8.0
 	github.com/tchap/go-patricia v2.3.0+incompatible // indirect
 	github.com/urfave/cli v1.22.9

go.sum

@@ -720,8 +720,8 @@ github.com/k3s-io/etcd/raft/v3 v3.5.4-k3s1 h1:wr4FPk1k51wyVmo5WFdU7PppvxgWkhTpVU
 github.com/k3s-io/etcd/raft/v3 v3.5.4-k3s1/go.mod h1:SCuunjYvZFC0fBX0vxMSPjuZmpcSk+XaAcMrD6Do03w=
 github.com/k3s-io/etcd/server/v3 v3.5.4-k3s1 h1:swbvfSDpl7QsYO6Vh+EBgxZCMyG4N1tUgzLPrIjTvVg=
 github.com/k3s-io/etcd/server/v3 v3.5.4-k3s1/go.mod h1:S5/YTU15KxymM5l3T6b09sNOHPXqGYIZStpuuGbb65c=
-github.com/k3s-io/helm-controller v0.13.0 h1:JfGEU6zrA6wBuVIBt73TbQYC3H+WDJaAc6Q5NbbaDwk=
-github.com/k3s-io/helm-controller v0.13.0/go.mod h1:f8aOuHQDpkshmUK/GiE+jJCJkUL8vp+EzCjV0uCFcsY=
+github.com/k3s-io/helm-controller v0.13.1 h1:eG2yZ0QzbtcfMe8GpTVtRtP6HgMDO/Pr9Q1EGbMKKCA=
+github.com/k3s-io/helm-controller v0.13.1/go.mod h1:f8aOuHQDpkshmUK/GiE+jJCJkUL8vp+EzCjV0uCFcsY=
 github.com/k3s-io/kine v0.9.6 h1:qomCtPrxIpFi09Q6JUDEbjWPjCliDgJ1Ns2N7l7aWxI=
 github.com/k3s-io/kine v0.9.6/go.mod h1:3N3AE7WgqbX4wYKJ9NdUItJ0i8koC+qaKbYc2sEaVns=
 github.com/k3s-io/klog v1.0.0-k3s2 h1:yyvD2bQbxG7m85/pvNctLX2bUDmva5kOBvuZ77tTGBA=

manifests/coredns.yaml

@@ -84,7 +84,7 @@ metadata:
     k8s-app: kube-dns
     kubernetes.io/name: "CoreDNS"
 spec:
-  #replicas: 1
+  revisionHistoryLimit: 0
   strategy:
     type: RollingUpdate
     rollingUpdate:

manifests/local-storage.yaml

@@ -41,7 +41,11 @@ metadata:
   name: local-path-provisioner
   namespace: kube-system
 spec:
-  replicas: 1
+  revisionHistoryLimit: 0
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
   selector:
     matchLabels:
       app: local-path-provisioner

manifests/metrics-server/metrics-server-deployment.yaml

@@ -13,6 +13,11 @@ metadata:
   labels:
     k8s-app: metrics-server
 spec:
+  revisionHistoryLimit: 0
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
   selector:
     matchLabels:
       k8s-app: metrics-server
@@ -39,7 +44,7 @@ spec:
         emptyDir: {}
       containers:
       - name: metrics-server
-        image: %{SYSTEM_DEFAULT_REGISTRY}%rancher/mirrored-metrics-server:v0.6.1
+        image: %{SYSTEM_DEFAULT_REGISTRY}%rancher/mirrored-metrics-server:v0.6.2
         args:
         - --cert-dir=/tmp
         - --secure-port=10250

manifests/traefik.yaml

@@ -5,15 +5,15 @@ metadata:
   name: traefik-crd
   namespace: kube-system
 spec:
-  chart: https://%{KUBERNETES_API}%/static/charts/traefik-crd-19.0.400.tgz
+  chart: https://%{KUBERNETES_API}%/static/charts/traefik-crd-20.3.1+up20.3.0.tgz
 ---
 apiVersion: helm.cattle.io/v1
 kind: HelmChart
 metadata:
   name: traefik
   namespace: kube-system
 spec:
-  chart: https://%{KUBERNETES_API}%/static/charts/traefik-19.0.400.tgz
+  chart: https://%{KUBERNETES_API}%/static/charts/traefik-20.3.1+up20.3.0.tgz
   set:
     global.systemDefaultRegistry: "%{SYSTEM_DEFAULT_REGISTRY_RAW}%"
   valuesContent: |-

pkg/agent/config/config.go

@@ -461,7 +461,6 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	}
 	nodeConfig.AgentConfig.Snapshotter = envInfo.Snapshotter
 	nodeConfig.AgentConfig.IPSECPSK = controlConfig.IPSECPSK
-	nodeConfig.AgentConfig.StrongSwanDir = filepath.Join(envInfo.DataDir, "agent", "strongswan")
 	nodeConfig.Containerd.Config = filepath.Join(envInfo.DataDir, "agent", "etc", "containerd", "config.toml")
 	nodeConfig.Containerd.Root = filepath.Join(envInfo.DataDir, "agent", "containerd")
 	if !nodeConfig.Docker && nodeConfig.ContainerRuntimeEndpoint == "" {

pkg/agent/flannel/setup.go

@@ -4,13 +4,12 @@ import (
 	"context"
 	"fmt"
 	"net"
-	"os"
+	"os/exec"
 	"path/filepath"
 	"strings"
 
 	"github.com/k3s-io/k3s/pkg/agent/util"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
-	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	v1 "k8s.io/api/core/v1"
@@ -229,12 +228,13 @@ func createFlannelConf(nodeConfig *config.Node) error {
 		backendConf = hostGWBackend
 	case config.FlannelBackendIPSEC:
 		backendConf = strings.ReplaceAll(ipsecBackend, "%psk%", nodeConfig.AgentConfig.IPSECPSK)
-		if err := setupStrongSwan(nodeConfig); err != nil {
-			return err
+		if _, err := exec.LookPath("swanctl"); err != nil {
+			return errors.Wrap(err, "k3s no longer includes strongswan - please install strongswan's swanctl and charon packages on your host")
 		}
+		logrus.Warnf("The ipsec backend is deprecated and will be removed in k3s v1.27; please switch to wireguard-native. Check our docs for information on how to migrate.")
 	case config.FlannelBackendWireguard:
 		backendConf = strings.ReplaceAll(wireguardBackend, "%flannelConfDir%", filepath.Dir(nodeConfig.FlannelConfFile))
-		logrus.Warnf("The wireguard backend is deprecated and will be removed in k3s v1.26, please switch to wireguard-native. Check our docs for information about how to migrate")
+		logrus.Warnf("The wireguard backend is deprecated and will be removed in k3s v1.26, please switch to wireguard-native. Check our docs for information about how to migrate.")
 	case config.FlannelBackendWireguardNative:
 		mode, ok := backendOptions["Mode"]
 		if !ok {
@@ -255,34 +255,6 @@ func createFlannelConf(nodeConfig *config.Node) error {
 	return util.WriteFile(nodeConfig.FlannelConfFile, confJSON)
 }
 
-func setupStrongSwan(nodeConfig *config.Node) error {
-	// if data dir env is not set point to root
-	dataDir := os.Getenv(version.ProgramUpper + "_DATA_DIR")
-	if dataDir == "" {
-		dataDir = "/"
-	}
-	dataDir = filepath.Join(dataDir, "etc", "strongswan")
-
-	info, err := os.Lstat(nodeConfig.AgentConfig.StrongSwanDir)
-	// something exists but is not a symlink, return
-	if err == nil && info.Mode()&os.ModeSymlink == 0 {
-		return nil
-	}
-	if err == nil {
-		target, err := os.Readlink(nodeConfig.AgentConfig.StrongSwanDir)
-		// current link is the same, return
-		if err == nil && target == dataDir {
-			return nil
-		}
-	}
-
-	// clean up strongswan old link
-	os.Remove(nodeConfig.AgentConfig.StrongSwanDir)
-
-	// make new strongswan link
-	return os.Symlink(dataDir, nodeConfig.AgentConfig.StrongSwanDir)
-}
-
 // fundNetMode returns the mode (ipv4, ipv6 or dual-stack) in which flannel is operating
 func findNetMode(cidrs []*net.IPNet) (int, error) {
 	dualStack, err := utilsnet.IsDualStackCIDRs(cidrs)

pkg/agent/run.go

@@ -253,7 +253,11 @@ func Run(ctx context.Context, cfg cmds.Agent) error {
 	}
 
 	if cfg.Rootless && !cfg.RootlessAlreadyUnshared {
-		if err := rootless.Rootless(cfg.DataDir); err != nil {
+		dualNode, err := utilsnet.IsDualStackIPStrings(cfg.NodeIP)
+		if err != nil {
+			return err
+		}
+		if err := rootless.Rootless(cfg.DataDir, dualNode); err != nil {
 			return err
 		}
 	}

pkg/cli/agent/agent.go

@@ -33,8 +33,8 @@ func Run(ctx *cli.Context) error {
 		return err
 	}
 
-	if os.Getuid() != 0 && runtime.GOOS != "windows" {
-		return fmt.Errorf("agent must be ran as root")
+	if runtime.GOOS != "windows" && os.Getuid() != 0 && !cmds.AgentConfig.Rootless {
+		return fmt.Errorf("agent must be run as root, or with --rootless")
 	}
 
 	if cmds.AgentConfig.TokenFile != "" {

pkg/cli/cmds/agent.go

@@ -276,6 +276,7 @@ func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command {
 			},
 			&SELinuxFlag,
 			LBServerPortFlag,
+			PreferBundledBin,
 
 			// Deprecated/hidden below
 

pkg/cli/cmds/root.go

@@ -17,6 +17,10 @@ var (
 		Destination: &Debug,
 		EnvVar:      version.ProgramUpper + "_DEBUG",
 	}
+	PreferBundledBin = cli.BoolFlag{
+		Name:  "prefer-bundled-bin",
+		Usage: "(experimental) Prefer bundled userspace binaries over host binaries",
+	}
 )
 
 func init() {