Merge pull request #4680 from manuelbuil/ha-verify-1.20

[Release 1.20] Verify new control plane nodes joining the cluster share the same config as cluster members

pkg/cluster/bootstrap.go

@@ -3,9 +3,11 @@ package cluster
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"errors"
 	"os"
 	"path/filepath"
+	"reflect"
 
 	"github.com/rancher/k3s/pkg/bootstrap"
 	"github.com/rancher/k3s/pkg/clientaccess"
@@ -134,6 +136,10 @@ func (c *Cluster) bootstrap(ctx context.Context) error {
 
 	// bootstrap managed database via HTTPS
 	if c.runtime.HTTPBootstrap {
+		// Assuming we should just compare on managed databases
+		if err := c.compareConfig(); err != nil {
+			return err
+		}
 		return c.httpBootstrap()
 	}
 
@@ -156,3 +162,30 @@ func (c *Cluster) Snapshot(ctx context.Context, config *config.Control) error {
 	}
 	return c.managedDB.Snapshot(ctx, config)
 }
+
+// compareConfig verifies that the config of the joining control plane node coincides with the cluster's config
+func (c *Cluster) compareConfig() error {
+	agentClientAccessInfo, err := clientaccess.ParseAndValidateTokenForUser(c.config.JoinURL, c.config.Token, "node")
+	if err != nil {
+		return err
+	}
+	serverConfig, err := clientaccess.Get("/v1-"+version.Program+"/config", agentClientAccessInfo)
+	if err != nil {
+		return err
+	}
+	clusterControl := &config.Control{}
+	if err := json.Unmarshal(serverConfig, clusterControl); err != nil {
+		return err
+	}
+
+	// We are saving IPs of ClusterIPRanges and ServiceIPRanges in 4-bytes representation but json decodes in 16-byte
+	c.config.CriticalControlArgs.ClusterIPRange.IP.To16()
+	c.config.CriticalControlArgs.ServiceIPRange.IP.To16()
+
+	if !reflect.DeepEqual(clusterControl.CriticalControlArgs, c.config.CriticalControlArgs) {
+		logrus.Debugf("This is the server CriticalControlArgs: %#v", clusterControl.CriticalControlArgs)
+		logrus.Debugf("This is the local CriticalControlArgs: %#v", c.config.CriticalControlArgs)
+		return errors.New("Unable to join cluster due to critical configuration value mismatch")
+	}
+	return nil
+}

pkg/daemons/config/types.go

@@ -93,7 +93,23 @@ type Agent struct {
 	ProtectKernelDefaults   bool
 }
 
+// CriticalControlArgs contains parameters that all control plane nodes in HA must share
+type CriticalControlArgs struct {
+	ClusterDNS       net.IP
+	ClusterDomain    string
+	ClusterIPRange   *net.IPNet
+	DisableCCM       bool
+	DisableKubeProxy bool
+	DisableNPC       bool
+	Disables         map[string]bool
+	FlannelBackend   string
+	NoCoreDNS        bool
+	ServiceIPRange   *net.IPNet
+	Skips            map[string]bool
+}
+
 type Control struct {
+	CriticalControlArgs
 	AdvertisePort int
 	AdvertiseIP   string
 	// The port which kubectl clients can access k8s
@@ -105,34 +121,23 @@ type Control struct {
 	APIServerBindAddress     string
 	AgentToken               string `json:"-"`
 	Token                    string `json:"-"`
-	ClusterIPRange           *net.IPNet
-	ServiceIPRange           *net.IPNet
 	ServiceNodePortRange     *utilnet.PortRange
-	ClusterDNS               net.IP
-	ClusterDomain            string
-	NoCoreDNS                bool
 	KubeConfigOutput         string
 	KubeConfigMode           string
 	DataDir                  string
-	Skips                    map[string]bool
-	Disables                 map[string]bool
 	Datastore                endpoint.Config
+	DisableAPIServer         bool
+	DisableControllerManager bool
+	DisableETCD              bool
+	DisableScheduler         bool
 	ExtraAPIArgs             []string
 	ExtraControllerArgs      []string
 	ExtraCloudControllerArgs []string
 	ExtraSchedulerAPIArgs    []string
 	NoLeaderElect            bool
 	JoinURL                  string
-	FlannelBackend           string
 	IPSECPSK                 string
 	DefaultLocalStoragePath  string
-	DisableCCM               bool
-	DisableNPC               bool
-	DisableKubeProxy         bool
-	DisableAPIServer         bool
-	DisableControllerManager bool
-	DisableScheduler         bool
-	DisableETCD              bool
 	ClusterInit              bool
 	ClusterReset             bool
 	ClusterResetRestorePath  string