Add tls for kine

Signed-off-by: Vitor Savian <[email protected]>

Bump kine

Signed-off-by: Vitor Savian <[email protected]>

Add integration tests for kine with tls

Signed-off-by: Vitor Savian <[email protected]>

go.mod

@@ -123,7 +123,7 @@ require (
 	github.com/ipfs/go-log/v2 v2.5.1
 	github.com/json-iterator/go v1.1.12
 	github.com/k3s-io/helm-controller v0.15.9
-	github.com/k3s-io/kine v0.11.4
+	github.com/k3s-io/kine v0.11.5
 	github.com/klauspost/compress v1.17.2
 	github.com/kubernetes-sigs/cri-tools v0.0.0-00010101000000-000000000000
 	github.com/lib/pq v1.10.2

go.sum

@@ -959,8 +959,8 @@ github.com/k3s-io/etcd/server/v3 v3.5.9-k3s1 h1:B3039IkTPnwQEt4tIMjC6yd6b1Q3Z9ZZ
 github.com/k3s-io/etcd/server/v3 v3.5.9-k3s1/go.mod h1:GgI1fQClQCFIzuVjlvdbMxNbnISt90gdfYyqiAIt65g=
 github.com/k3s-io/helm-controller v0.15.9 h1:eBZq0KkZCDyWh4og+tyI43Nt9T5TNjc7QCFhAt1aR64=
 github.com/k3s-io/helm-controller v0.15.9/go.mod h1:AYitg40howLjKloL/zdjDDOPL1jg/K5R4af0tQcyPR8=
-github.com/k3s-io/kine v0.11.4 h1:ZIXQT4vPPKNL9DwLF4dQ11tWtpJ1C/7OKNIpFmTkImo=
-github.com/k3s-io/kine v0.11.4/go.mod h1:NmwOWsWgB3aScq5+LEYytAaceqkG7lmCLLjjrWug8v4=
+github.com/k3s-io/kine v0.11.5 h1:/oqepXJblSL3sYLhYDa22mRhrA3CVO6DzKcipnFdO8U=
+github.com/k3s-io/kine v0.11.5/go.mod h1:NmwOWsWgB3aScq5+LEYytAaceqkG7lmCLLjjrWug8v4=
 github.com/k3s-io/klog/v2 v2.100.1-k3s1 h1:xb/Ta8dpQuIZueQEw2YTZUYrKoILdBmPiITVkNmYPa0=
 github.com/k3s-io/klog/v2 v2.100.1-k3s1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
 github.com/k3s-io/kube-router/v2 v2.0.1 h1:UCsdkQjSfOkVakixilRDDkG9yq775GBSKxBfsyUj8ng=

pkg/cli/cmds/server.go

@@ -60,6 +60,7 @@ type Server struct {
 	DatastoreCAFile          string
 	DatastoreCertFile        string
 	DatastoreKeyFile         string
+	DatastoreTLS             bool
 	AdvertiseIP              string
 	AdvertisePort            int
 	DisableScheduler         bool
@@ -310,6 +311,11 @@ var ServerFlags = []cli.Flag{
 		Usage: "(flags) Customized flag for kube-cloud-controller-manager process",
 		Value: &ServerConfig.ExtraCloudControllerArgs,
 	},
+	&cli.BoolFlag{
+		Name:        "datastore-tls",
+		Usage:       "(db) Enable tls for the datastore",
+		Destination: &ServerConfig.DatastoreTLS,
+	},
 	&cli.StringFlag{
 		Name:        "datastore-endpoint",
 		Usage:       "(db) Specify etcd, NATS, MySQL, Postgres, or SQLite (default) data source name",

pkg/cli/server/server.go

@@ -49,9 +49,7 @@ func RunWithControllers(app *cli.Context, leaderControllers server.CustomControl
 }
 
 func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomControllers, controllers server.CustomControllers) error {
-	var (
-		err error
-	)
+	var err error
 	// Validate build env
 	cmds.MustValidateGolang()
 
@@ -154,6 +152,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 	serverConfig.ControlConfig.Datastore.BackendTLSConfig.CAFile = cfg.DatastoreCAFile
 	serverConfig.ControlConfig.Datastore.BackendTLSConfig.CertFile = cfg.DatastoreCertFile
 	serverConfig.ControlConfig.Datastore.BackendTLSConfig.KeyFile = cfg.DatastoreKeyFile
+	serverConfig.ControlConfig.DatastoreTLS = cfg.DatastoreTLS
 	serverConfig.ControlConfig.AdvertiseIP = cfg.AdvertiseIP
 	serverConfig.ControlConfig.AdvertisePort = cfg.AdvertisePort
 	serverConfig.ControlConfig.FlannelBackend = cfg.FlannelBackend

pkg/cluster/cluster.go

@@ -84,7 +84,7 @@ func (c *Cluster) Start(ctx context.Context) (<-chan struct{}, error) {
 		return nil, err
 	}
 
-	if err := c.startStorage(ctx); err != nil {
+	if err := c.startStorage(ctx, false); err != nil {
 		return nil, err
 	}
 
@@ -132,12 +132,19 @@ func (c *Cluster) Start(ctx context.Context) (<-chan struct{}, error) {
 // This calls into the kine endpoint code, which sets up the database client
 // and unix domain socket listener if using an external database. In the case of an etcd
 // backend it just returns the user-provided etcd endpoints and tls config.
-func (c *Cluster) startStorage(ctx context.Context) error {
-	if c.storageStarted {
+func (c *Cluster) startStorage(ctx context.Context, bootstrap bool) error {
+	if c.storageStarted && !c.config.DatastoreTLS {
 		return nil
 	}
 	c.storageStarted = true
 
+	if !bootstrap {
+		// set the tls config for the kine storage
+		c.config.Datastore.ServerTLSConfig.CAFile = c.config.Runtime.ETCDServerCA
+		c.config.Datastore.ServerTLSConfig.CertFile = c.config.Runtime.ServerETCDCert
+		c.config.Datastore.ServerTLSConfig.KeyFile = c.config.Runtime.ServerETCDKey
+	}
+
 	// start listening on the kine socket as an etcd endpoint, or return the external etcd endpoints
 	etcdConfig, err := endpoint.Listen(ctx, c.config.Datastore)
 	if err != nil {
@@ -148,9 +155,15 @@ func (c *Cluster) startStorage(ctx context.Context) error {
 	// based on what the kine wrapper tells us about the datastore. Single-node datastores like sqlite don't require
 	// leader election, while basically all others (etcd, external database, etc) do since they allow multiple servers.
 	c.config.Runtime.EtcdConfig = etcdConfig
-	c.config.Datastore.BackendTLSConfig = etcdConfig.TLSConfig
-	c.config.Datastore.Endpoint = strings.Join(etcdConfig.Endpoints, ",")
-	c.config.NoLeaderElect = !etcdConfig.LeaderElect
+
+	// after the bootstrap we need to set the args for api-server with kine in unixs or just set the
+	// values if the datastoreTLS is not enabled
+	if !bootstrap || !c.config.DatastoreTLS {
+		c.config.Datastore.BackendTLSConfig = etcdConfig.TLSConfig
+		c.config.Datastore.Endpoint = strings.Join(etcdConfig.Endpoints, ",")
+		c.config.NoLeaderElect = !etcdConfig.LeaderElect
+	}
+
 	return nil
 }
 

pkg/cluster/storage.go

@@ -20,7 +20,6 @@ import (
 const maxBootstrapWaitAttempts = 5
 
 func RotateBootstrapToken(ctx context.Context, config *config.Control, oldToken string) error {
-
 	token, err := util.ReadTokenFromFile(config.Runtime.ServerToken, config.Runtime.ServerCA, config.DataDir)
 	if err != nil {
 		return err
@@ -151,7 +150,7 @@ func bootstrapKeyData(ctx context.Context, storageClient client.Client) (*client
 // bootstrap key as a lock. This function will not return successfully until either the
 // bootstrap key has been locked, or data is read into the struct.
 func (c *Cluster) storageBootstrap(ctx context.Context) error {
-	if err := c.startStorage(ctx); err != nil {
+	if err := c.startStorage(ctx, true); err != nil {
 		return err
 	}
 

pkg/daemons/config/types.go

@@ -179,6 +179,7 @@ type Control struct {
 	KubeConfigMode           string
 	HelmJobImage             string
 	DataDir                  string
+	DatastoreTLS             bool
 	Datastore                endpoint.Config `json:"-"`
 	Disables                 map[string]bool
 	DisableAPIServer         bool

pkg/daemons/control/deps/deps.go

@@ -42,8 +42,7 @@ const (
 	RequestHeaderCN = "system:auth-proxy"
 )
 
-var (
-	kubeconfigTemplate = template.Must(template.New("kubeconfig").Parse(`apiVersion: v1
+var kubeconfigTemplate = template.Must(template.New("kubeconfig").Parse(`apiVersion: v1
 clusters:
 - cluster:
     server: {{.URL}}
@@ -64,7 +63,6 @@ users:
     client-certificate: {{.ClientCert}}
     client-key: {{.ClientKey}}
 `))
-)
 
 func migratePassword(p *passwd.Passwd) error {
 	server, _ := p.Pass("server")
@@ -283,9 +281,7 @@ func genEncryptedNetworkInfo(controlConfig *config.Control) error {
 }
 
 func getServerPass(passwd *passwd.Passwd, config *config.Control) (string, error) {
-	var (
-		err error
-	)
+	var err error
 
 	serverPass := config.Token
 	if serverPass == "" {
@@ -448,14 +444,16 @@ func genServerCerts(config *config.Control) error {
 }
 
 func genETCDCerts(config *config.Control) error {
-
 	runtime := config.Runtime
 	regen, err := createSigningCertKey("etcd-server", runtime.ETCDServerCA, runtime.ETCDServerCAKey)
 	if err != nil {
 		return err
 	}
 
-	altNames := &certutil.AltNames{}
+	altNames := &certutil.AltNames{
+		DNSNames: []string{"kine.sock"},
+	}
+
 	addSANs(altNames, config.SANs)
 
 	if _, err := createClientCertKey(regen, "etcd-client", nil,
@@ -845,5 +843,4 @@ func genCloudConfig(controlConfig *config.Control) error {
 		return err
 	}
 	return os.WriteFile(controlConfig.Runtime.CloudControllerConfig, b, 0600)
-
 }

pkg/daemons/control/server.go

@@ -286,7 +286,6 @@ func prepare(ctx context.Context, config *config.Control) error {
 	deps.CreateRuntimeCertFiles(config)
 
 	cluster := cluster.New(config)
-
 	if err := cluster.Bootstrap(ctx, config.ClusterReset); err != nil {
 		return err
 	}

tests/integration/startup/startup_int_test.go

@@ -1,6 +1,7 @@
 package integration
 
 import (
+	"errors"
 	"os"
 	"path/filepath"
 	"testing"
@@ -13,9 +14,11 @@ import (
 	. "github.com/onsi/gomega/gstruct"
 )
 
-var startupServer *testutil.K3sServer
-var startupServerArgs = []string{}
-var testLock int
+var (
+	startupServer     *testutil.K3sServer
+	startupServerArgs = []string{}
+	testLock          int
+)
 
 var _ = BeforeSuite(func() {
 	if testutil.IsExistingServer() {
@@ -27,7 +30,6 @@ var _ = BeforeSuite(func() {
 })
 
 var _ = Describe("startup tests", Ordered, func() {
-
 	When("a default server is created", func() {
 		It("is created with no arguments", func() {
 			var err error
@@ -39,6 +41,59 @@ var _ = Describe("startup tests", Ordered, func() {
 				return testutil.K3sDefaultDeployments()
 			}, "120s", "5s").Should(Succeed())
 		})
+		It("has kine without tls", func() {
+			Eventually(func() error {
+				match, err := testutil.SearchK3sLog(startupServer, "Kine available at unix://kine.sock")
+				if err != nil {
+					return err
+				}
+				if match {
+					return nil
+				}
+				return errors.New("error finding kine sock")
+			}, "30s", "2s").Should(Succeed())
+		})
+		It("does not use kine with tls after bootstrap", func() {
+			Eventually(func() error {
+				match, err := testutil.SearchK3sLog(startupServer, "Kine available at unixs://kine.sock")
+				if err != nil {
+					return err
+				}
+				if match {
+					return errors.New("Kine with tls when the datastore-tls is not set")
+				}
+				return nil
+			}, "30s", "2s").Should(Succeed())
+		})
+		It("dies cleanly", func() {
+			Expect(testutil.K3sKillServer(startupServer)).To(Succeed())
+			Expect(testutil.K3sCleanup(-1, "")).To(Succeed())
+		})
+	})
+	When("a server with datastore-tls is created", func() {
+		It("is created with datastore-tls", func() {
+			var err error
+			startupServerArgs = []string{"--datastore-tls"}
+			startupServer, err = testutil.K3sStartServer(startupServerArgs...)
+			Expect(err).ToNot(HaveOccurred())
+		})
+		It("has the default pods deployed", func() {
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "120s", "5s").Should(Succeed())
+		})
+		It("set kine to use tls", func() {
+			Eventually(func() error {
+				match, err := testutil.SearchK3sLog(startupServer, "Kine available at unixs://kine.sock")
+				if err != nil {
+					return err
+				}
+				if match {
+					return nil
+				}
+				return errors.New("error finding unixs://kine.sock")
+			}, "30s", "2s").Should(Succeed())
+		})
 		It("dies cleanly", func() {
 			Expect(testutil.K3sKillServer(startupServer)).To(Succeed())
 			Expect(testutil.K3sCleanup(-1, "")).To(Succeed())
@@ -263,13 +318,14 @@ var _ = Describe("startup tests", Ordered, func() {
 			Expect(testutil.K3sCleanup(-1, "")).To(Succeed())
 		})
 	})
-
 })
 
-var failed bool
-var _ = AfterEach(func() {
-	failed = failed || CurrentSpecReport().Failed()
-})
+var (
+	failed bool
+	_      = AfterEach(func() {
+		failed = failed || CurrentSpecReport().Failed()
+	})
+)
 
 var _ = AfterSuite(func() {
 	if !testutil.IsExistingServer() {