diff --git a/cmd/kubeadm/app/cmd/join.go b/cmd/kubeadm/app/cmd/join.go index f5ea1e7f9b860..b9ea65c04fefb 100644 --- a/cmd/kubeadm/app/cmd/join.go +++ b/cmd/kubeadm/app/cmd/join.go @@ -48,6 +48,7 @@ import ( kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet" markmasterphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/markmaster" patchnodephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode" + uploadconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig" "k8s.io/kubernetes/cmd/kubeadm/app/preflight" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" @@ -300,21 +301,22 @@ func (j *Join) Run(out io.Writer) error { } // If the node should host a new control plane instance + var initConfiguration *kubeadmapi.InitConfiguration if j.cfg.ControlPlane == true { // Retrives the kubeadm configuration used during kubeadm init glog.V(1).Infoln("[join] retrieving KubeConfig objects") - clusterConfiguration, err := j.FetchInitClusterConfiguration(tlsBootstrapCfg) + initConfiguration, err = j.FetchInitConfiguration(tlsBootstrapCfg) if err != nil { return err } - // injects into the kubeadm configuration used for init the information about the joining node - clusterConfiguration.NodeRegistration = j.cfg.NodeRegistration - clusterConfiguration.APIEndpoint.AdvertiseAddress = j.cfg.APIEndpoint.AdvertiseAddress + // injects into the kubeadm configuration the information about the joining node + initConfiguration.NodeRegistration = j.cfg.NodeRegistration + initConfiguration.APIEndpoint = j.cfg.APIEndpoint // Checks if the cluster configuration supports // joining a new control plane instance and if all the necessary certificates are provided - if err = j.CheckIfReadyForAdditionalControlPlane(clusterConfiguration); err != nil { + if err = j.CheckIfReadyForAdditionalControlPlane(initConfiguration); err != nil { // outputs the not ready for hosting a new control plane instance message ctx := map[string]string{ "Error": err.Error(), @@ -327,11 +329,11 @@ func (j *Join) Run(out io.Writer) error { // run kubeadm init preflight checks for checking all the prequisites glog.Infoln("[join] running pre-flight checks before initializing the new control plane instance") - preflight.RunInitMasterChecks(utilsexec.New(), clusterConfiguration, j.ignorePreflightErrors) + preflight.RunInitMasterChecks(utilsexec.New(), initConfiguration, j.ignorePreflightErrors) // Prepares the node for hosting a new control plane instance by writing necessary // KubeConfig files, and static pod manifests - if err = j.PrepareForHostingControlPlane(clusterConfiguration); err != nil { + if err = j.PrepareForHostingControlPlane(initConfiguration); err != nil { return err } } @@ -348,8 +350,8 @@ func (j *Join) Run(out io.Writer) error { // if the node is hosting a new control plane instance if j.cfg.ControlPlane == true { - // Marks the node with master taint and label. - if err := j.MarkMaster(); err != nil { + // Completes the control plane setup + if err := j.PostInstallControlPlane(initConfiguration); err != nil { return err } @@ -367,52 +369,48 @@ func (j *Join) Run(out io.Writer) error { return nil } -// FetchInitClusterConfiguration reads the cluster configuration from the kubeadm-admin configMap, -func (j *Join) FetchInitClusterConfiguration(tlsBootstrapCfg *clientcmdapi.Config) (*kubeadmapi.InitConfiguration, error) { +// FetchInitConfiguration reads the cluster configuration from the kubeadm-admin configMap, +func (j *Join) FetchInitConfiguration(tlsBootstrapCfg *clientcmdapi.Config) (*kubeadmapi.InitConfiguration, error) { // creates a client to access the cluster using the bootstrap token identity tlsClient, err := kubeconfigutil.ToClientSet(tlsBootstrapCfg) if err != nil { return nil, errors.Wrap(err, "Unable to access the cluster") } - // Fetches the cluster configuration - kubeadmConfig, err := configutil.FetchConfigFromFileOrCluster(tlsClient, os.Stdout, "join", "") + // Fetches the init configuration + initConfiguration, err := configutil.FetchConfigFromFileOrCluster(tlsClient, os.Stdout, "join", "", true) if err != nil { return nil, errors.Wrap(err, "Unable to fetch the kubeadm-config ConfigMap") } - // Converts public API struct to internal API - clusterConfiguration := &kubeadmapi.InitConfiguration{} - kubeadmscheme.Scheme.Convert(kubeadmConfig, clusterConfiguration, nil) - - return clusterConfiguration, nil + return initConfiguration, nil } // CheckIfReadyForAdditionalControlPlane ensures that the cluster is in a state that supports // joining an additional control plane instance and if the node is ready to join -func (j *Join) CheckIfReadyForAdditionalControlPlane(clusterConfiguration *kubeadmapi.InitConfiguration) error { +func (j *Join) CheckIfReadyForAdditionalControlPlane(initConfiguration *kubeadmapi.InitConfiguration) error { // blocks if the cluster was created without a stable control plane endpoint - if clusterConfiguration.ControlPlaneEndpoint == "" { + if initConfiguration.ControlPlaneEndpoint == "" { return fmt.Errorf("unable to add a new control plane instance a cluster that doesn't have a stable controlPlaneEndpoint address") } // blocks if the cluster was created without an external etcd cluster - if clusterConfiguration.Etcd.External == nil { + if initConfiguration.Etcd.External == nil { return fmt.Errorf("unable to add a new control plane instance on a cluster that doesn't use an external etcd") } // blocks if control plane is self-hosted - if features.Enabled(clusterConfiguration.FeatureGates, features.SelfHosting) { + if features.Enabled(initConfiguration.FeatureGates, features.SelfHosting) { return fmt.Errorf("self-hosted clusters are deprecated and won't be supported by `kubeadm join --experimental-control-plane`") } // blocks if the certificates for the control plane are stored in secrets (instead of the local pki folder) - if features.Enabled(clusterConfiguration.FeatureGates, features.StoreCertsInSecrets) { + if features.Enabled(initConfiguration.FeatureGates, features.StoreCertsInSecrets) { return fmt.Errorf("certificates stored in secrets, as well as self-hosted clusters are deprecated and won't be supported by `kubeadm join --experimental-control-plane`") } // checks if the certificates that must be equal across contolplane instances are provided - if ret, err := certsphase.SharedCertificateExists(clusterConfiguration); !ret { + if ret, err := certsphase.SharedCertificateExists(initConfiguration); !ret { return err } @@ -420,28 +418,28 @@ func (j *Join) CheckIfReadyForAdditionalControlPlane(clusterConfiguration *kubea } // PrepareForHostingControlPlane makes all preparation activities require for a node hosting a new control plane instance -func (j *Join) PrepareForHostingControlPlane(clusterConfiguration *kubeadmapi.InitConfiguration) error { +func (j *Join) PrepareForHostingControlPlane(initConfiguration *kubeadmapi.InitConfiguration) error { // Creates the admin kubeconfig file for the admin and for kubeadm itself. - if err := kubeconfigphase.CreateAdminKubeConfigFile(kubeadmconstants.KubernetesDir, clusterConfiguration); err != nil { + if err := kubeconfigphase.CreateAdminKubeConfigFile(kubeadmconstants.KubernetesDir, initConfiguration); err != nil { return errors.Wrap(err, "error generating the admin kubeconfig file") } // Generate missing certificates (if any) - if err := certsphase.CreatePKIAssets(clusterConfiguration); err != nil { + if err := certsphase.CreatePKIAssets(initConfiguration); err != nil { return err } // Generate kubeconfig files for controller manager, scheduler and for the admin/kubeadm itself // NB. The kubeconfig file for kubelet will be generated by the TLS bootstrap process in // following steps of the join --experimental-control plane workflow - if err := kubeconfigphase.CreateJoinControlPlaneKubeConfigFiles(kubeadmconstants.KubernetesDir, clusterConfiguration); err != nil { + if err := kubeconfigphase.CreateJoinControlPlaneKubeConfigFiles(kubeadmconstants.KubernetesDir, initConfiguration); err != nil { return errors.Wrap(err, "error generating kubeconfig files") } // Creates static pod manifests file for the control plane components to be deployed on this node // Static pods will be created and managed by the kubelet as soon as it starts - if err := controlplanephase.CreateInitStaticPodManifestFiles(kubeadmconstants.GetStaticPodDirectory(), clusterConfiguration); err != nil { + if err := controlplanephase.CreateInitStaticPodManifestFiles(kubeadmconstants.GetStaticPodDirectory(), initConfiguration); err != nil { return errors.Wrap(err, "error creating static pod manifest files for the control plane components") } @@ -488,8 +486,10 @@ func (j *Join) BootstrapKubelet(tlsBootstrapCfg *clientcmdapi.Config) error { return err } - // Write env file with flags for the kubelet to use. Also register taints - if err := kubeletphase.WriteKubeletDynamicEnvFile(&j.cfg.NodeRegistration, j.cfg.FeatureGates, true, kubeadmconstants.KubeletRunDirectory); err != nil { + // Write env file with flags for the kubelet to use. We do not need to write the --register-with-taints for the master, + // as we handle that ourselves in the markmaster phase + // TODO: Maybe we want to do that some time in the future, in order to remove some logic from the markmaster phase? + if err := kubeletphase.WriteKubeletDynamicEnvFile(&j.cfg.NodeRegistration, j.cfg.FeatureGates, false, kubeadmconstants.KubeletRunDirectory); err != nil { return err } @@ -527,8 +527,8 @@ func (j *Join) BootstrapKubelet(tlsBootstrapCfg *clientcmdapi.Config) error { return nil } -// MarkMaster marks the new node as master -func (j *Join) MarkMaster() error { +// PostInstallControlPlane marks the new node as master and update the cluster status with information about current node +func (j *Join) PostInstallControlPlane(initConfiguration *kubeadmapi.InitConfiguration) error { kubeConfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.AdminKubeConfigFileName) client, err := kubeconfigutil.ClientSetFromFile(kubeConfigFile) @@ -536,8 +536,13 @@ func (j *Join) MarkMaster() error { return errors.Wrap(err, "couldn't create kubernetes client") } - err = markmasterphase.MarkMaster(client, j.cfg.NodeRegistration.Name, j.cfg.NodeRegistration.Taints) - if err != nil { + glog.V(1).Info("[join] uploading currently used configuration to the cluster") + if err := uploadconfigphase.UploadConfiguration(initConfiguration, client); err != nil { + return fmt.Errorf("error uploading configuration: %v", err) + } + + glog.V(1).Info("[join] marking the master with right label") + if err = markmasterphase.MarkMaster(client, initConfiguration.NodeRegistration.Name, initConfiguration.NodeRegistration.Taints); err != nil { return errors.Wrap(err, "error applying master label and taints") } diff --git a/cmd/kubeadm/app/cmd/upgrade/common.go b/cmd/kubeadm/app/cmd/upgrade/common.go index ef880874c0f1b..3698e832a98ba 100644 --- a/cmd/kubeadm/app/cmd/upgrade/common.go +++ b/cmd/kubeadm/app/cmd/upgrade/common.go @@ -64,7 +64,7 @@ func enforceRequirements(flags *applyPlanFlags, dryRun bool, newK8sVersion strin // Fetch the configuration from a file or ConfigMap and validate it fmt.Println("[upgrade/config] Making sure the configuration is correct:") - cfg, err := configutil.FetchConfigFromFileOrCluster(client, os.Stdout, "upgrade/config", flags.cfgPath) + cfg, err := configutil.FetchConfigFromFileOrCluster(client, os.Stdout, "upgrade/config", flags.cfgPath, false) if err != nil { if apierrors.IsNotFound(err) { fmt.Printf("[upgrade/config] In order to upgrade, a ConfigMap called %q in the %s namespace must exist.\n", constants.InitConfigurationConfigMap, metav1.NamespaceSystem) diff --git a/cmd/kubeadm/app/cmd/upgrade/node.go b/cmd/kubeadm/app/cmd/upgrade/node.go index 8758c7736cb3b..dca43e4f53f40 100644 --- a/cmd/kubeadm/app/cmd/upgrade/node.go +++ b/cmd/kubeadm/app/cmd/upgrade/node.go @@ -150,10 +150,6 @@ func NewCmdUpgradeControlPlane() *cobra.Command { options.AddKubeConfigFlag(cmd.Flags(), &flags.kubeConfigPath) cmd.Flags().BoolVar(&flags.dryRun, "dry-run", flags.dryRun, "Do not change any state, just output the actions that would be performed.") - - //TODO: following values should retrieved form the kubeadm-config config map; remove as soon as the new config wil be in place - cmd.Flags().StringVar(&flags.advertiseAddress, "apiserver-advertise-address", flags.advertiseAddress, "If the node is joining as a master, the IP address the API Server will advertise it's listening on.") - cmd.Flags().StringVar(&flags.nodeName, "node-name", flags.nodeName, "Specify the node name.") return cmd } @@ -231,16 +227,11 @@ func RunUpgradeControlPlane(flags *controlplaneUpgradeFlags) error { waiter := apiclient.NewKubeWaiter(client, upgrade.UpgradeManifestTimeout, os.Stdout) // Fetches the cluster configuration - cfg, err := configutil.FetchConfigFromFileOrCluster(client, os.Stdout, "upgrade", "") + cfg, err := configutil.FetchConfigFromFileOrCluster(client, os.Stdout, "upgrade", "", false) if err != nil { return fmt.Errorf("Unable to fetch the kubeadm-config ConfigMap: %v", err) } - //TODO: as soon as the new config wil be in place check if the node is a known control plane instance - // and retrive corresponding infos (now are temporary managed as flag) - cfg.NodeRegistration.Name = flags.nodeName - cfg.APIEndpoint.AdvertiseAddress = flags.advertiseAddress - // Rotate API server certificate if needed if err := upgrade.BackupAPIServerCertIfNeeded(cfg, flags.dryRun); err != nil { return fmt.Errorf("Unable to rotate API server certificate: %v", err) diff --git a/cmd/kubeadm/app/componentconfigs/BUILD b/cmd/kubeadm/app/componentconfigs/BUILD index 0c9a22b732dd0..63333949164a4 100644 --- a/cmd/kubeadm/app/componentconfigs/BUILD +++ b/cmd/kubeadm/app/componentconfigs/BUILD @@ -3,6 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", srcs = [ + "config.go", "defaults.go", "registrations.go", "scheme.go", @@ -21,12 +22,14 @@ go_library( "//pkg/proxy/apis/config:go_default_library", "//pkg/proxy/apis/config/v1alpha1:go_default_library", "//pkg/proxy/apis/config/validation:go_default_library", + "//pkg/util/version:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/validation/field:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/kube-proxy/config/v1alpha1:go_default_library", "//vendor/k8s.io/utils/pointer:go_default_library", ], @@ -48,14 +51,22 @@ filegroup( go_test( name = "go_default_test", - srcs = ["validation_test.go"], + srcs = [ + "config_test.go", + "validation_test.go", + ], embed = [":go_default_library"], deps = [ "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", + "//cmd/kubeadm/app/util/apiclient:go_default_library", "//pkg/kubelet/apis/config:go_default_library", "//pkg/proxy/apis/config:go_default_library", + "//pkg/util/version:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", "//vendor/k8s.io/utils/pointer:go_default_library", ], ) diff --git a/cmd/kubeadm/app/componentconfigs/config.go b/cmd/kubeadm/app/componentconfigs/config.go new file mode 100644 index 0000000000000..df4443cd5c13f --- /dev/null +++ b/cmd/kubeadm/app/componentconfigs/config.go @@ -0,0 +1,80 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package componentconfigs + +import ( + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + clientset "k8s.io/client-go/kubernetes" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" + kubeproxyconfig "k8s.io/kubernetes/pkg/proxy/apis/config" + "k8s.io/kubernetes/pkg/util/version" +) + +// GetFromKubeletConfigMap returns the pointer to the ComponentConfig API object read from the kubelet-config-version +// ConfigMap map stored in the cluster +func GetFromKubeletConfigMap(client clientset.Interface, version *version.Version) (runtime.Object, error) { + + // Read the ConfigMap from the cluster based on what version the kubelet is + configMapName := kubeadmconstants.GetKubeletConfigMapName(version) + kubeletCfg, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(configMapName, metav1.GetOptions{}) + if err != nil { + return nil, err + } + + kubeletConfigData, ok := kubeletCfg.Data[kubeadmconstants.KubeletBaseConfigurationConfigMapKey] + if !ok { + return nil, fmt.Errorf("unexpected error when reading %s ConfigMap: %s key value pair missing", configMapName, kubeadmconstants.KubeletBaseConfigurationConfigMapKey) + } + + // Decodes the kubeletConfigData into the internal component config + obj := &kubeletconfig.KubeletConfiguration{} + err = unmarshalObject(obj, []byte(kubeletConfigData)) + if err != nil { + return nil, err + } + + return obj, nil +} + +// GetFromKubeProxyConfigMap returns the pointer to the ComponentConfig API object read from the kube-proxy +// ConfigMap map stored in the cluster +func GetFromKubeProxyConfigMap(client clientset.Interface, version *version.Version) (runtime.Object, error) { + + // Read the ConfigMap from the cluster + kubeproxyCfg, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(kubeadmconstants.KubeProxyConfigMap, metav1.GetOptions{}) + if err != nil { + return nil, err + } + + kubeproxyConfigData, ok := kubeproxyCfg.Data[kubeadmconstants.KubeProxyConfigMapKey] + if !ok { + return nil, fmt.Errorf("unexpected error when reading %s ConfigMap: %s key value pair missing", kubeadmconstants.KubeProxyConfigMap, kubeadmconstants.KubeProxyConfigMapKey) + } + + // Decodes the Config map dat into the internal component config + obj := &kubeproxyconfig.KubeProxyConfiguration{} + err = unmarshalObject(obj, []byte(kubeproxyConfigData)) + if err != nil { + return nil, err + } + + return obj, nil +} diff --git a/cmd/kubeadm/app/componentconfigs/config_test.go b/cmd/kubeadm/app/componentconfigs/config_test.go new file mode 100644 index 0000000000000..9372e6f46efe3 --- /dev/null +++ b/cmd/kubeadm/app/componentconfigs/config_test.go @@ -0,0 +1,146 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package componentconfigs + +import ( + "testing" + + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + clientsetfake "k8s.io/client-go/kubernetes/fake" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" + "k8s.io/kubernetes/pkg/util/version" +) + +var cfgFiles = map[string][]byte{ + "Kube-proxy_componentconfig": []byte(` +apiVersion: kubeproxy.config.k8s.io/v1alpha1 +kind: KubeProxyConfiguration +`), + "Kubelet_componentconfig": []byte(` +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +`), +} + +func TestGetFromConfigMap(t *testing.T) { + k8sVersion := version.MustParseGeneric("v1.12.0") + + var tests = []struct { + name string + component RegistrationKind + configMap *fakeConfigMap + expectedError bool + }{ + { + name: "valid kube-proxy", + component: KubeProxyConfigurationKind, + configMap: &fakeConfigMap{ + name: kubeadmconstants.KubeProxyConfigMap, + data: map[string]string{ + kubeadmconstants.KubeProxyConfigMapKey: string(cfgFiles["Kube-proxy_componentconfig"]), + }, + }, + }, + { + name: "invalid kube-proxy - missing ConfigMap", + component: KubeProxyConfigurationKind, + configMap: nil, + expectedError: true, + }, + { + name: "invalid kube-proxy - missing key", + component: KubeProxyConfigurationKind, + configMap: &fakeConfigMap{ + name: kubeadmconstants.KubeProxyConfigMap, + data: map[string]string{}, + }, + expectedError: true, + }, + { + name: "valid kubelet", + component: KubeletConfigurationKind, + configMap: &fakeConfigMap{ + name: kubeadmconstants.GetKubeletConfigMapName(k8sVersion), + data: map[string]string{ + kubeadmconstants.KubeletBaseConfigurationConfigMapKey: string(cfgFiles["Kubelet_componentconfig"]), + }, + }, + }, + { + name: "invalid kubelet - missing ConfigMap", + component: KubeletConfigurationKind, + configMap: nil, + expectedError: true, + }, + { + name: "invalid kubelet - missing key", + component: KubeletConfigurationKind, + configMap: &fakeConfigMap{ + name: kubeadmconstants.GetKubeletConfigMapName(k8sVersion), + data: map[string]string{}, + }, + expectedError: true, + }, + } + + for _, rt := range tests { + t.Run(rt.name, func(t2 *testing.T) { + client := clientsetfake.NewSimpleClientset() + + if rt.configMap != nil { + err := rt.configMap.create(client) + if err != nil { + t.Errorf("unexpected create ConfigMap %s", rt.configMap.name) + return + } + } + + registration := Known[rt.component] + + obj, err := registration.GetFromConfigMap(client, k8sVersion) + if rt.expectedError != (err != nil) { + t.Errorf("unexpected return err from GetFromConfigMap: %v", err) + return + } + if rt.expectedError { + return + } + + if obj == nil { + t.Error("unexpected nil return value") + } + }) + } +} + +type fakeConfigMap struct { + name string + data map[string]string +} + +func (c *fakeConfigMap) create(client clientset.Interface) error { + return apiclient.CreateOrUpdateConfigMap(client, &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: c.name, + Namespace: metav1.NamespaceSystem, + }, + Data: c.data, + }) +} diff --git a/cmd/kubeadm/app/componentconfigs/registrations.go b/cmd/kubeadm/app/componentconfigs/registrations.go index bae83ac435657..f1703e34fe45e 100644 --- a/cmd/kubeadm/app/componentconfigs/registrations.go +++ b/cmd/kubeadm/app/componentconfigs/registrations.go @@ -20,12 +20,14 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/validation/field" + clientset "k8s.io/client-go/kubernetes" kubeproxyconfigv1alpha1 "k8s.io/kube-proxy/config/v1alpha1" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" kubeletconfigv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/config/v1beta1" kubeproxyconfig "k8s.io/kubernetes/pkg/proxy/apis/config" + "k8s.io/kubernetes/pkg/util/version" ) // AddToSchemeFunc is a function that adds known types and API GroupVersions to a scheme @@ -47,6 +49,8 @@ type Registration struct { GetFromInternalConfig func(*kubeadmapi.ClusterConfiguration) (runtime.Object, bool) // SetToInternalConfig sets the pointer to a ComponentConfig API object embedded in the internal kubeadm config struct SetToInternalConfig func(runtime.Object, *kubeadmapi.ClusterConfiguration) bool + // GetFromConfigMap returns the pointer to the ComponentConfig API object read from the config map stored in the cluster + GetFromConfigMap func(clientset.Interface, *version.Version) (runtime.Object, error) } // Marshal marshals obj to bytes for the current Registration @@ -60,13 +64,20 @@ func (r Registration) Unmarshal(fileContent []byte) (runtime.Object, error) { obj := r.EmptyValue.DeepCopyObject() // Decode the file content into obj which is a pointer to an empty struct of the internal ComponentConfig - // object, using the componentconfig Codecs that knows about all APIs - if err := runtime.DecodeInto(Codecs.UniversalDecoder(), fileContent, obj); err != nil { + if err := unmarshalObject(obj, fileContent); err != nil { return nil, err } return obj, nil } +func unmarshalObject(obj runtime.Object, fileContent []byte) error { + // Decode the file content using the componentconfig Codecs that knows about all APIs + if err := runtime.DecodeInto(Codecs.UniversalDecoder(), fileContent, obj); err != nil { + return err + } + return nil +} + const ( // KubeletConfigurationKind is the kind for the kubelet ComponentConfig KubeletConfigurationKind RegistrationKind = "KubeletConfiguration" @@ -99,6 +110,7 @@ var Known Registrations = map[RegistrationKind]Registration{ } return ok }, + GetFromConfigMap: GetFromKubeProxyConfigMap, }, KubeletConfigurationKind: { MarshalGroupVersion: kubeletconfigv1beta1.SchemeGroupVersion, @@ -116,6 +128,7 @@ var Known Registrations = map[RegistrationKind]Registration{ } return ok }, + GetFromConfigMap: GetFromKubeletConfigMap, }, } diff --git a/cmd/kubeadm/app/constants/constants.go b/cmd/kubeadm/app/constants/constants.go index 3176f52cd7d79..63063297d3806 100644 --- a/cmd/kubeadm/app/constants/constants.go +++ b/cmd/kubeadm/app/constants/constants.go @@ -154,6 +154,8 @@ const ( MastersGroup = "system:masters" // NodesGroup defines the well-known group for all nodes. NodesGroup = "system:nodes" + // NodesUserPrefix defines the user name prefix as requested by the Node authorizer. + NodesUserPrefix = "system:node:" // NodesClusterRoleBinding defines the well-known ClusterRoleBinding which binds the too permissive system:node // ClusterRole to the system:nodes group. Since kubeadm is using the Node Authorizer, this ClusterRoleBinding's // system:nodes group subject is removed if present. @@ -187,13 +189,24 @@ const ( AnnotationKubeadmCRISocket = "kubeadm.alpha.kubernetes.io/cri-socket" // InitConfigurationConfigMap specifies in what ConfigMap in the kube-system namespace the `kubeadm init` configuration should be stored - // TODO: Rename this to ClusterConfigurationConfigMap + // TODO: Rename this to KubeadmConfigConfigMap InitConfigurationConfigMap = "kubeadm-config" // InitConfigurationConfigMapKey specifies in what ConfigMap key the master configuration should be stored - // TODO: Rename this to ClusterConfigurationConfigMapKey, and migrate the value to ClusterConfiguration, - // but still support the old MasterConfiguration naming in earlier versions - InitConfigurationConfigMapKey = "InitConfiguration" + // TODO: This was used in v1.11 with vi1alpha2 config and older. Remove in v1.13 + InitConfigurationConfigMapKey = "MasterConfiguration" + + // ClusterConfigurationConfigMapKey specifies in what ConfigMap key the cluster configuration should be stored + ClusterConfigurationConfigMapKey = "ClusterConfiguration" + + // ClusterStatusConfigMapKey specifies in what ConfigMap key the cluster status should be stored + ClusterStatusConfigMapKey = "ClusterStatus" + + // KubeProxyConfigMap specifies in what ConfigMap in the kube-system namespace the kube-proxy configuration should be stored + KubeProxyConfigMap = "kube-proxy" + + // KubeProxyConfigMapKey specifies in what ConfigMap key the component config of kube-proxy should be stored + KubeProxyConfigMapKey = "config.conf" // KubeletBaseConfigurationConfigMapPrefix specifies in what ConfigMap in the kube-system namespace the initial remote configuration of kubelet should be stored KubeletBaseConfigurationConfigMapPrefix = "kubelet-config-" @@ -460,3 +473,8 @@ func GetDNSVersion(dnsType string) string { return KubeDNSVersion } } + +// GetKubeletConfigMapName returns the right ConfigMap name for the right branch of k8s +func GetKubeletConfigMapName(k8sVersion *version.Version) string { + return fmt.Sprintf("%s%d.%d", KubeletBaseConfigurationConfigMapPrefix, k8sVersion.Major(), k8sVersion.Minor()) +} diff --git a/cmd/kubeadm/app/phases/addons/proxy/BUILD b/cmd/kubeadm/app/phases/addons/proxy/BUILD index 2c731992655f7..4bb1949b32ea6 100644 --- a/cmd/kubeadm/app/phases/addons/proxy/BUILD +++ b/cmd/kubeadm/app/phases/addons/proxy/BUILD @@ -39,6 +39,7 @@ go_library( "//cmd/kubeadm/app/images:go_default_library", "//cmd/kubeadm/app/util:go_default_library", "//cmd/kubeadm/app/util/apiclient:go_default_library", + "//pkg/apis/rbac/v1:go_default_library", "//staging/src/k8s.io/api/apps/v1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/rbac/v1:go_default_library", diff --git a/cmd/kubeadm/app/phases/addons/proxy/manifests.go b/cmd/kubeadm/app/phases/addons/proxy/manifests.go index a425bb679c68a..93a3bf1dc6f45 100644 --- a/cmd/kubeadm/app/phases/addons/proxy/manifests.go +++ b/cmd/kubeadm/app/phases/addons/proxy/manifests.go @@ -22,7 +22,7 @@ const ( kind: ConfigMap apiVersion: v1 metadata: - name: kube-proxy + name: {{ .ProxyConfigMap }} namespace: kube-system labels: app: kube-proxy @@ -46,7 +46,7 @@ data: - name: default user: tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token - config.conf: |- + {{ .ProxyConfigMapKey }}: |- {{ .ProxyConfig}} ` @@ -79,7 +79,7 @@ spec: imagePullPolicy: IfNotPresent command: - /usr/local/bin/kube-proxy - - --config=/var/lib/kube-proxy/config.conf + - --config=/var/lib/kube-proxy/{{ .ProxyConfigMapKey }} securityContext: privileged: true volumeMounts: @@ -96,7 +96,7 @@ spec: volumes: - name: kube-proxy configMap: - name: kube-proxy + name: {{ .ProxyConfigMap }} - name: xtables-lock hostPath: path: /run/xtables.lock diff --git a/cmd/kubeadm/app/phases/addons/proxy/proxy.go b/cmd/kubeadm/app/phases/addons/proxy/proxy.go index aea3729a77308..862a89fa4188e 100644 --- a/cmd/kubeadm/app/phases/addons/proxy/proxy.go +++ b/cmd/kubeadm/app/phases/addons/proxy/proxy.go @@ -33,6 +33,7 @@ import ( "k8s.io/kubernetes/cmd/kubeadm/app/images" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" + rbachelper "k8s.io/kubernetes/pkg/apis/rbac/v1" ) const ( @@ -65,17 +66,23 @@ func EnsureProxyAddon(cfg *kubeadmapi.InitConfiguration, client clientset.Interf var proxyConfigMapBytes, proxyDaemonSetBytes []byte proxyConfigMapBytes, err = kubeadmutil.ParseTemplate(KubeProxyConfigMap19, struct { - MasterEndpoint string - ProxyConfig string + MasterEndpoint string + ProxyConfig string + ProxyConfigMap string + ProxyConfigMapKey string }{ - MasterEndpoint: masterEndpoint, - ProxyConfig: prefixBytes.String(), + MasterEndpoint: masterEndpoint, + ProxyConfig: prefixBytes.String(), + ProxyConfigMap: constants.KubeProxyConfigMap, + ProxyConfigMapKey: constants.KubeProxyConfigMapKey, }) if err != nil { return fmt.Errorf("error when parsing kube-proxy configmap template: %v", err) } - proxyDaemonSetBytes, err = kubeadmutil.ParseTemplate(KubeProxyDaemonSet19, struct{ Image string }{ - Image: images.GetKubeControlPlaneImage(constants.KubeProxy, &cfg.ClusterConfiguration), + proxyDaemonSetBytes, err = kubeadmutil.ParseTemplate(KubeProxyDaemonSet19, struct{ Image, ProxyConfigMap, ProxyConfigMapKey string }{ + Image: images.GetKubeControlPlaneImage(constants.KubeProxy, &cfg.ClusterConfiguration), + ProxyConfigMap: constants.KubeProxyConfigMap, + ProxyConfigMapKey: constants.KubeProxyConfigMapKey, }) if err != nil { return fmt.Errorf("error when parsing kube-proxy daemonset template: %v", err) @@ -128,7 +135,7 @@ func createKubeProxyAddon(configMapBytes, daemonSetbytes []byte, client clientse } func createClusterRoleBindings(client clientset.Interface) error { - return apiclient.CreateOrUpdateClusterRoleBinding(client, &rbac.ClusterRoleBinding{ + if err := apiclient.CreateOrUpdateClusterRoleBinding(client, &rbac.ClusterRoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: "kubeadm:node-proxier", }, @@ -144,5 +151,39 @@ func createClusterRoleBindings(client clientset.Interface) error { Namespace: metav1.NamespaceSystem, }, }, + }); err != nil { + return err + } + + // Create a role for granting read only access to the kube-proxy component config ConfigMap + if err := apiclient.CreateOrUpdateRole(client, &rbac.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: constants.KubeProxyConfigMap, + Namespace: metav1.NamespaceSystem, + }, + Rules: []rbac.PolicyRule{ + rbachelper.NewRule("get").Groups("").Resources("configmaps").Names(constants.KubeProxyConfigMap).RuleOrDie(), + }, + }); err != nil { + return err + } + + // Bind the role to bootstrap tokens for allowing fetchConfiguration during join + return apiclient.CreateOrUpdateRoleBinding(client, &rbac.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: constants.KubeProxyConfigMap, + Namespace: metav1.NamespaceSystem, + }, + RoleRef: rbac.RoleRef{ + APIGroup: rbac.GroupName, + Kind: "Role", + Name: constants.KubeProxyConfigMap, + }, + Subjects: []rbac.Subject{ + { + Kind: rbac.GroupKind, + Name: constants.NodeBootstrapTokenAuthGroup, + }, + }, }) } diff --git a/cmd/kubeadm/app/phases/addons/proxy/proxy_test.go b/cmd/kubeadm/app/phases/addons/proxy/proxy_test.go index 517c5be7f7a34..ff74040d82b69 100644 --- a/cmd/kubeadm/app/phases/addons/proxy/proxy_test.go +++ b/cmd/kubeadm/app/phases/addons/proxy/proxy_test.go @@ -99,17 +99,21 @@ func TestCompileManifests(t *testing.T) { { manifest: KubeProxyConfigMap19, data: struct { - MasterEndpoint, ProxyConfig string + MasterEndpoint, ProxyConfig, ProxyConfigMap, ProxyConfigMapKey string }{ - MasterEndpoint: "foo", - ProxyConfig: " bindAddress: 0.0.0.0\n clusterCIDR: 192.168.1.1\n enableProfiling: false", + MasterEndpoint: "foo", + ProxyConfig: " bindAddress: 0.0.0.0\n clusterCIDR: 192.168.1.1\n enableProfiling: false", + ProxyConfigMap: "bar", + ProxyConfigMapKey: "baz", }, expected: true, }, { manifest: KubeProxyDaemonSet19, - data: struct{ Image string }{ - Image: "foo", + data: struct{ Image, ProxyConfigMap, ProxyConfigMapKey string }{ + Image: "foo", + ProxyConfigMap: "bar", + ProxyConfigMapKey: "baz", }, expected: true, }, diff --git a/cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go b/cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go index c7a09a50457e1..f4a90d455efbc 100644 --- a/cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go +++ b/cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go @@ -174,7 +174,7 @@ func getKubeConfigSpecs(cfg *kubeadmapi.InitConfiguration) (map[string]*kubeConf kubeadmconstants.KubeletKubeConfigFileName: { CACert: caCert, APIServer: masterEndpoint, - ClientName: fmt.Sprintf("system:node:%s", cfg.NodeRegistration.Name), + ClientName: fmt.Sprintf("%s%s", kubeadmconstants.NodesUserPrefix, cfg.NodeRegistration.Name), ClientCertAuth: &clientCertAuth{ CAKey: caKey, Organizations: []string{kubeadmconstants.NodesGroup}, diff --git a/cmd/kubeadm/app/phases/kubeconfig/kubeconfig_test.go b/cmd/kubeadm/app/phases/kubeconfig/kubeconfig_test.go index 92c9ef346dd87..0e884e3f17f46 100644 --- a/cmd/kubeadm/app/phases/kubeconfig/kubeconfig_test.go +++ b/cmd/kubeadm/app/phases/kubeconfig/kubeconfig_test.go @@ -119,7 +119,7 @@ func TestGetKubeConfigSpecs(t *testing.T) { }, { kubeConfigFile: kubeadmconstants.KubeletKubeConfigFileName, - clientName: fmt.Sprintf("system:node:%s", cfg.NodeRegistration.Name), + clientName: fmt.Sprintf("%s%s", kubeadmconstants.NodesUserPrefix, cfg.NodeRegistration.Name), organizations: []string{kubeadmconstants.NodesGroup}, }, { diff --git a/cmd/kubeadm/app/phases/kubelet/config.go b/cmd/kubeadm/app/phases/kubelet/config.go index be0d85bee6a73..87dea28597690 100644 --- a/cmd/kubeadm/app/phases/kubelet/config.go +++ b/cmd/kubeadm/app/phases/kubelet/config.go @@ -56,7 +56,7 @@ func CreateConfigMap(cfg *kubeadmapi.InitConfiguration, client clientset.Interfa return err } - configMapName := configMapName(k8sVersion) + configMapName := kubeadmconstants.GetKubeletConfigMapName(k8sVersion) fmt.Printf("[kubelet] Creating a ConfigMap %q in namespace %s with the configuration for the kubelets in the cluster\n", configMapName, metav1.NamespaceSystem) kubeletBytes, err := getConfigBytes(cfg.ComponentConfigs.Kubelet) @@ -90,7 +90,7 @@ func createConfigMapRBACRules(client clientset.Interface, k8sVersion *version.Ve Namespace: metav1.NamespaceSystem, }, Rules: []rbac.PolicyRule{ - rbachelper.NewRule("get").Groups("").Resources("configmaps").Names(configMapName(k8sVersion)).RuleOrDie(), + rbachelper.NewRule("get").Groups("").Resources("configmaps").Names(kubeadmconstants.GetKubeletConfigMapName(k8sVersion)).RuleOrDie(), }, }); err != nil { return err @@ -124,7 +124,7 @@ func createConfigMapRBACRules(client clientset.Interface, k8sVersion *version.Ve func DownloadConfig(client clientset.Interface, kubeletVersion *version.Version, kubeletDir string) error { // Download the ConfigMap from the cluster based on what version the kubelet is - configMapName := configMapName(kubeletVersion) + configMapName := kubeadmconstants.GetKubeletConfigMapName(kubeletVersion) fmt.Printf("[kubelet] Downloading configuration for the kubelet from the %q ConfigMap in the %s namespace\n", configMapName, metav1.NamespaceSystem) @@ -142,11 +142,6 @@ func DownloadConfig(client clientset.Interface, kubeletVersion *version.Version, return writeConfigBytesToDisk([]byte(kubeletCfg.Data[kubeadmconstants.KubeletBaseConfigurationConfigMapKey]), kubeletDir) } -// configMapName returns the right ConfigMap name for the right branch of k8s -func configMapName(k8sVersion *version.Version) string { - return fmt.Sprintf("%s%d.%d", kubeadmconstants.KubeletBaseConfigurationConfigMapPrefix, k8sVersion.Major(), k8sVersion.Minor()) -} - // configMapRBACName returns the name for the Role/RoleBinding for the kubelet config configmap for the right branch of k8s func configMapRBACName(k8sVersion *version.Version) string { return fmt.Sprintf("%s%d.%d", kubeadmconstants.KubeletBaseConfigMapRolePrefix, k8sVersion.Major(), k8sVersion.Minor()) diff --git a/cmd/kubeadm/app/phases/kubelet/dynamic.go b/cmd/kubeadm/app/phases/kubelet/dynamic.go index 602182a76035f..f16131b075afe 100644 --- a/cmd/kubeadm/app/phases/kubelet/dynamic.go +++ b/cmd/kubeadm/app/phases/kubelet/dynamic.go @@ -32,7 +32,7 @@ import ( // This func is ONLY run if the user enables the `DynamicKubeletConfig` feature gate, which is by default off func EnableDynamicConfigForNode(client clientset.Interface, nodeName string, kubeletVersion *version.Version) error { - configMapName := configMapName(kubeletVersion) + configMapName := kubeadmconstants.GetKubeletConfigMapName(kubeletVersion) fmt.Printf("[kubelet] Enabling Dynamic Kubelet Config for Node %q; config sourced from ConfigMap %q in namespace %s\n", nodeName, configMapName, metav1.NamespaceSystem) fmt.Println("[kubelet] WARNING: The Dynamic Kubelet Config feature is beta, but off by default. It hasn't been well-tested yet at this stage, use with caution.") diff --git a/cmd/kubeadm/app/phases/uploadconfig/BUILD b/cmd/kubeadm/app/phases/uploadconfig/BUILD index 77201659d64aa..797439ba33aa1 100644 --- a/cmd/kubeadm/app/phases/uploadconfig/BUILD +++ b/cmd/kubeadm/app/phases/uploadconfig/BUILD @@ -12,13 +12,16 @@ go_library( importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig", deps = [ "//cmd/kubeadm/app/apis/kubeadm:go_default_library", + "//cmd/kubeadm/app/apis/kubeadm/scheme:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", "//cmd/kubeadm/app/util/apiclient:go_default_library", "//cmd/kubeadm/app/util/config:go_default_library", "//pkg/apis/rbac/v1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/rbac/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", ], ) @@ -45,10 +48,13 @@ go_test( "//cmd/kubeadm/app/apis/kubeadm/scheme:go_default_library", "//cmd/kubeadm/app/apis/kubeadm/v1alpha3:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", + "//cmd/kubeadm/app/util/apiclient:go_default_library", "//cmd/kubeadm/app/util/config:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", "//staging/src/k8s.io/client-go/testing:go_default_library", ], diff --git a/cmd/kubeadm/app/phases/uploadconfig/uploadconfig.go b/cmd/kubeadm/app/phases/uploadconfig/uploadconfig.go index 528cbfb2254b8..a13d7dbdb2634 100644 --- a/cmd/kubeadm/app/phases/uploadconfig/uploadconfig.go +++ b/cmd/kubeadm/app/phases/uploadconfig/uploadconfig.go @@ -21,9 +21,12 @@ import ( "k8s.io/api/core/v1" rbac "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" clientset "k8s.io/client-go/kubernetes" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" @@ -39,20 +42,36 @@ const ( // UploadConfiguration saves the InitConfiguration used for later reference (when upgrading for instance) func UploadConfiguration(cfg *kubeadmapi.InitConfiguration, client clientset.Interface) error { - fmt.Printf("[uploadconfig] storing the configuration used in ConfigMap %q in the %q Namespace\n", kubeadmconstants.InitConfigurationConfigMap, metav1.NamespaceSystem) + // Prepare the ClusterConfiguration for upload + // The components store their config in their own ConfigMaps, then reset the .ComponentConfig struct; // We don't want to mutate the cfg itself, so create a copy of it using .DeepCopy of it first - clusterConfigToUpload := cfg.ClusterConfiguration.DeepCopy() - // TODO: Reset the .ComponentConfig struct like this: - // cfgToUpload.ComponentConfigs = kubeadmapi.ComponentConfigs{} - // in order to not upload any other components' config to the kubeadm-config - // ConfigMap. The components store their config in their own ConfigMaps. - // Before this line can be uncommented util/config.loadConfigurationBytes() - // needs to support reading the different components' ConfigMaps first. - - // Marshal the object into YAML - cfgYaml, err := configutil.MarshalKubeadmConfigObject(clusterConfigToUpload) + clusterConfigurationToUpload := cfg.ClusterConfiguration.DeepCopy() + clusterConfigurationToUpload.ComponentConfigs = kubeadmapi.ComponentConfigs{} + + // Marshal the ClusterConfiguration into YAML + clusterConfigurationYaml, err := configutil.MarshalKubeadmConfigObject(clusterConfigurationToUpload) + if err != nil { + return err + } + + // Prepare the ClusterStatus for upload + // Gets the current cluster status + // TODO: use configmap locks on this object on the get before the update. + clusterStatus, err := getClusterStatus(client) + if err != nil { + return err + } + + // Updates the ClusterStatus with the current control plane instance + if clusterStatus.APIEndpoints == nil { + clusterStatus.APIEndpoints = map[string]kubeadmapi.APIEndpoint{} + } + clusterStatus.APIEndpoints[cfg.NodeRegistration.Name] = cfg.APIEndpoint + + // Marshal the ClusterStatus back into into YAML + clusterStatusYaml, err := configutil.MarshalKubeadmConfigObject(clusterStatus) if err != nil { return err } @@ -63,7 +82,8 @@ func UploadConfiguration(cfg *kubeadmapi.InitConfiguration, client clientset.Int Namespace: metav1.NamespaceSystem, }, Data: map[string]string{ - kubeadmconstants.InitConfigurationConfigMapKey: string(cfgYaml), + kubeadmconstants.ClusterConfigurationConfigMapKey: string(clusterConfigurationYaml), + kubeadmconstants.ClusterStatusConfigMapKey: string(clusterStatusYaml), }, }) if err != nil { @@ -109,3 +129,22 @@ func UploadConfiguration(cfg *kubeadmapi.InitConfiguration, client clientset.Int }, }) } + +func getClusterStatus(client clientset.Interface) (*kubeadmapi.ClusterStatus, error) { + obj := &kubeadmapi.ClusterStatus{} + + // Read the ConfigMap from the cluster + configMap, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(kubeadmconstants.InitConfigurationConfigMap, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return obj, nil + } + if err != nil { + return nil, err + } + + // Decode the file content using the componentconfig Codecs that knows about all APIs + if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), []byte(configMap.Data[kubeadmconstants.ClusterStatusConfigMapKey]), obj); err != nil { + return nil, err + } + return obj, nil +} diff --git a/cmd/kubeadm/app/phases/uploadconfig/uploadconfig_test.go b/cmd/kubeadm/app/phases/uploadconfig/uploadconfig_test.go index 9da16f921186c..19fcb6c5e37f3 100644 --- a/cmd/kubeadm/app/phases/uploadconfig/uploadconfig_test.go +++ b/cmd/kubeadm/app/phases/uploadconfig/uploadconfig_test.go @@ -20,15 +20,18 @@ import ( "reflect" "testing" + "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + clientset "k8s.io/client-go/kubernetes" clientsetfake "k8s.io/client-go/kubernetes/fake" core "k8s.io/client-go/testing" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" kubeadmapiv1alpha3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha3" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" ) @@ -89,6 +92,12 @@ func TestUploadConfiguration(t *testing.T) { t2.Fatalf("UploadConfiguration() error = %v", err) } + status := &kubeadmapi.ClusterStatus{ + APIEndpoints: map[string]kubeadmapi.APIEndpoint{ + "node-foo": cfg.APIEndpoint, + }, + } + client := clientsetfake.NewSimpleClientset() if tt.errOnCreate != nil { client.PrependReactor("create", "configmaps", func(action core.Action) (bool, runtime.Object, error) { @@ -114,9 +123,9 @@ func TestUploadConfiguration(t *testing.T) { if err != nil { t2.Fatalf("Fail to query ConfigMap error = %v", err) } - configData := masterCfg.Data[kubeadmconstants.InitConfigurationConfigMapKey] + configData := masterCfg.Data[kubeadmconstants.ClusterConfigurationConfigMapKey] if configData == "" { - t2.Fatalf("Fail to find ConfigMap key") + t2.Fatal("Fail to find ClusterConfigurationConfigMapKey key") } decodedCfg := &kubeadmapi.ClusterConfiguration{} @@ -127,7 +136,80 @@ func TestUploadConfiguration(t *testing.T) { if !reflect.DeepEqual(decodedCfg, &cfg.ClusterConfiguration) { t2.Errorf("the initial and decoded ClusterConfiguration didn't match") } + + statusData := masterCfg.Data[kubeadmconstants.ClusterStatusConfigMapKey] + if statusData == "" { + t2.Fatal("failed to find ClusterStatusConfigMapKey key") + } + + decodedStatus := &kubeadmapi.ClusterStatus{} + if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), []byte(statusData), decodedStatus); err != nil { + t2.Fatalf("unable to decode status from bytes: %v", err) + } + + if !reflect.DeepEqual(decodedStatus, status) { + t2.Error("the initial and decoded ClusterStatus didn't match") + } } }) } } + +func TestGetClusterStatus(t *testing.T) { + var tests = []struct { + name string + clusterStatus *kubeadmapi.ClusterStatus + expectedClusterEndpoints int + }{ + { + name: "return empty ClusterStatus if cluster kubeadm-config doesn't exist (e.g init)", + expectedClusterEndpoints: 0, + }, + { + name: "return ClusterStatus if cluster kubeadm-config exist (e.g upgrade)", + clusterStatus: &kubeadmapi.ClusterStatus{ + APIEndpoints: map[string]kubeadmapi.APIEndpoint{ + "dummy": {AdvertiseAddress: "1.2.3.4", BindPort: 1234}, + }, + }, + expectedClusterEndpoints: 1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client := clientsetfake.NewSimpleClientset() + + if tt.clusterStatus != nil { + createConfigMapWithStatus(tt.clusterStatus, client) + } + + actual, err := getClusterStatus(client) + if err != nil { + t.Error("GetClusterStatus returned unexpected error") + return + } + if tt.expectedClusterEndpoints != len(actual.APIEndpoints) { + t.Error("actual ClusterStatus doesn't return expected endpoints") + } + }) + } +} + +// createConfigMapWithStatus create a ConfigMap with ClusterStatus for TestGetClusterStatus +func createConfigMapWithStatus(statusToCreate *kubeadmapi.ClusterStatus, client clientset.Interface) error { + statusYaml, err := configutil.MarshalKubeadmConfigObject(statusToCreate) + if err != nil { + return err + } + + return apiclient.CreateOrUpdateConfigMap(client, &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: kubeadmconstants.InitConfigurationConfigMap, + Namespace: metav1.NamespaceSystem, + }, + Data: map[string]string{ + kubeadmconstants.ClusterStatusConfigMapKey: string(statusYaml), + }, + }) +} diff --git a/cmd/kubeadm/app/util/config/BUILD b/cmd/kubeadm/app/util/config/BUILD index 70e324bb7aede..a38912c6737aa 100644 --- a/cmd/kubeadm/app/util/config/BUILD +++ b/cmd/kubeadm/app/util/config/BUILD @@ -26,13 +26,14 @@ go_library( "//pkg/util/node:go_default_library", "//pkg/util/version:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/tools/bootstrap/token/util:go_default_library", + "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", + "//staging/src/k8s.io/client-go/util/cert:go_default_library", "//vendor/github.com/golang/glog:go_default_library", ], ) @@ -54,6 +55,7 @@ go_test( "//cmd/kubeadm/app/constants:go_default_library", "//cmd/kubeadm/app/util:go_default_library", "//cmd/kubeadm/app/util/apiclient:go_default_library", + "//pkg/util/version:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", diff --git a/cmd/kubeadm/app/util/config/cluster.go b/cmd/kubeadm/app/util/config/cluster.go index caff1d9787f58..7917381e7c48b 100644 --- a/cmd/kubeadm/app/util/config/cluster.go +++ b/cmd/kubeadm/app/util/config/cluster.go @@ -17,60 +17,244 @@ limitations under the License. package config import ( + "crypto/x509" + "errors" "fmt" "io" "io/ioutil" + "path/filepath" + "strings" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + certutil "k8s.io/client-go/util/cert" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" + "k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs" "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/pkg/util/version" ) // FetchConfigFromFileOrCluster fetches configuration required for upgrading your cluster from a file (which has precedence) or a ConfigMap in the cluster -// TODO: This func should be renamed FetchClusterConfigFromFileOrCluster, and return a ClusterConfiguration instead of an InitConfiguration -func FetchConfigFromFileOrCluster(client clientset.Interface, w io.Writer, logPrefix, cfgPath string) (*kubeadmapi.InitConfiguration, error) { +func FetchConfigFromFileOrCluster(client clientset.Interface, w io.Writer, logPrefix, cfgPath string, newControlPlane bool) (*kubeadmapi.InitConfiguration, error) { // Load the configuration from a file or the cluster - configBytes, err := loadConfigurationBytes(client, w, logPrefix, cfgPath) + initcfg, err := loadConfiguration(client, w, logPrefix, cfgPath, newControlPlane) if err != nil { return nil, err } - // Unmarshal the versioned configuration populated from the file or ConfigMap, convert it to the internal API types, then default and validate - initcfg, err := BytesToInternalConfig(configBytes) - if err != nil { - return nil, err - } - - //TODO: this will be reviewed in the following PR for reading/storing the kubeadm-config ConfigMap + // Apply dynamic defaults if err := SetInitDynamicDefaults(initcfg); err != nil { return nil, err } return initcfg, err } -// loadConfigurationBytes loads the configuration byte slice from either a file or the cluster ConfigMap -func loadConfigurationBytes(client clientset.Interface, w io.Writer, logPrefix, cfgPath string) ([]byte, error) { +// loadConfiguration loads the configuration byte slice from either a file or the cluster ConfigMap +func loadConfiguration(client clientset.Interface, w io.Writer, logPrefix, cfgPath string, newControlPlane bool) (*kubeadmapi.InitConfiguration, error) { // The config file has the highest priority if cfgPath != "" { fmt.Fprintf(w, "[%s] Reading configuration options from a file: %s\n", logPrefix, cfgPath) - return ioutil.ReadFile(cfgPath) + return loadInitConfigurationFromFile(cfgPath) } fmt.Fprintf(w, "[%s] Reading configuration from the cluster...\n", logPrefix) + fmt.Fprintf(w, "[%s] FYI: You can look at this config file with 'kubectl -n %s get cm %s -oyaml'\n", logPrefix, metav1.NamespaceSystem, constants.InitConfigurationConfigMap) + return getInitConfigurationFromCluster(constants.KubernetesDir, client, newControlPlane) +} +func loadInitConfigurationFromFile(cfgPath string) (*kubeadmapi.InitConfiguration, error) { + configBytes, err := ioutil.ReadFile(cfgPath) + if err != nil { + return nil, err + } + + // Unmarshal the versioned configuration populated from the file, + // convert it to the internal API types, then default and validate + // NB the file can be one of + // - a single YAML, with a v1alpha2.MasterConfiguration object (with embedded component configs) + // - multiple YAML, with a combination of + // - a YAML with a v1alpha3.InitConfiguration object + // - a YAML with a v1alpha3.ClusterConfiguration object (without embedded component configs) + // - separated YAML for components configs + initcfg, err := BytesToInternalConfig(configBytes) + if err != nil { + return nil, err + } + + return initcfg, nil +} + +func getInitConfigurationFromCluster(kubeconfigDir string, client clientset.Interface, newControlPlane bool) (*kubeadmapi.InitConfiguration, error) { // TODO: This code should support reading the MasterConfiguration key as well for backwards-compat - // Also, the key really should be ClusterConfiguration... + // Also, the config map really should be KubeadmConfigConfigMap... configMap, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(constants.InitConfigurationConfigMap, metav1.GetOptions{}) - if apierrors.IsNotFound(err) { - // Return the apierror directly so the caller of this function can know what type of error occurred and act based on that - return []byte{}, err - } else if err != nil { - return []byte{}, fmt.Errorf("an unexpected error happened when trying to get the ConfigMap %q in the %s namespace: %v", constants.InitConfigurationConfigMap, metav1.NamespaceSystem, err) + if err != nil { + return nil, err } - // TODO: Load the kube-proxy and kubelet ComponentConfig ConfigMaps here as different YAML documents and append to the byte slice - fmt.Fprintf(w, "[%s] FYI: You can look at this config file with 'kubectl -n %s get cm %s -oyaml'\n", logPrefix, metav1.NamespaceSystem, constants.InitConfigurationConfigMap) - return []byte(configMap.Data[constants.InitConfigurationConfigMapKey]), nil + // TODO: remove in V1.13 + // If InitConfigurationConfigMapKey exist, the kubeadm-config was created with v1.11 + if _, ok := configMap.Data[constants.InitConfigurationConfigMapKey]; ok { + return getInitConfigurationFromConfigMapV11(configMap.Data) + } + + return getInitConfigurationFromConfigMaps(kubeconfigDir, client, configMap.Data, newControlPlane) +} + +func getInitConfigurationFromConfigMapV11(data map[string]string) (*kubeadmapi.InitConfiguration, error) { + configBytes := []byte(data[constants.InitConfigurationConfigMapKey]) + + // Unmarshal the versioned configuration populated from the file, + // convert it to the internal API types, then default and validate + // NB the config map created with v11 is a single YAML, with a v1alpha2.MasterConfiguration object (with embedded component configs) + initcfg, err := BytesToInternalConfig(configBytes) + if err != nil { + return nil, err + } + + return initcfg, nil +} + +func getInitConfigurationFromConfigMaps(kubeconfigDir string, client clientset.Interface, data map[string]string, newControlPlane bool) (*kubeadmapi.InitConfiguration, error) { + // In case of cluster crated with v1.12 InitConfiguration is composed with data from different places + initcfg := &kubeadmapi.InitConfiguration{} + + // gets ClusterConfiguration from kubeadm-config + clusterConfigurationData, ok := data[constants.ClusterConfigurationConfigMapKey] + if !ok { + return nil, fmt.Errorf("unexpected error when reading kubeadm-config ConfigMap: %s key value pair missing", constants.ClusterConfigurationConfigMapKey) + } + if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), []byte(clusterConfigurationData), &initcfg.ClusterConfiguration); err != nil { + return nil, err + } + + // gets the component configs from the corresponding config maps + if err := getComponentConfigs(client, &initcfg.ClusterConfiguration); err != nil { + return nil, err + } + + // if this isn't a new controlplane instance (e.g. in case of kubeadm upgrades) + // get nodes specific information as well + if !newControlPlane { + // gets the nodeRegistration for the current from the node object + if err := getNodeRegistration(kubeconfigDir, client, &initcfg.NodeRegistration); err != nil { + return nil, err + } + // gets the APIEndpoint for the current node from then ClusterStatus in the kubeadm-config ConfigMap + if err := getAPIEndpoint(data, initcfg.NodeRegistration.Name, &initcfg.APIEndpoint); err != nil { + return nil, err + } + } + + return initcfg, nil +} + +// getNodeRegistration returns the nodeRegistration for the current node +func getNodeRegistration(kubeconfigDir string, client clientset.Interface, nodeRegistration *kubeadmapi.NodeRegistrationOptions) error { + // gets the name of the current node + nodeName, err := getNodeNameFromKubeletConfig(kubeconfigDir) + if err != nil { + return err + } + + // gets the corresponding node and retrives attributes stored there. + node, err := client.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{}) + if err != nil { + return err + } + + criSocket, ok := node.ObjectMeta.Annotations[constants.AnnotationKubeadmCRISocket] + if !ok { + return fmt.Errorf("Node %s doesn't have %s annotation", nodeName, constants.AnnotationKubeadmCRISocket) + } + + // returns the nodeRegistration attributes + nodeRegistration.Name = nodeName + nodeRegistration.CRISocket = criSocket + nodeRegistration.Taints = node.Spec.Taints + // NB. currently nodeRegistration.KubeletExtraArgs isn't stored at node level but only in the kubeadm-flags.env + // that isn't modified during upgrades + // in future we might reconsider this thus enabling changes to the kubeadm-flags.env during upgrades as well + return nil +} + +// getNodeNameFromConfig gets the node name from a kubelet config file +// TODO: in future we want to switch to a more canonical way for doing this e.g. by having this +// information in the local kubelet config.yaml +func getNodeNameFromKubeletConfig(kubeconfigDir string) (string, error) { + // loads the kubelet.conf file + fileName := filepath.Join(kubeconfigDir, constants.KubeletKubeConfigFileName) + config, err := clientcmd.LoadFromFile(fileName) + if err != nil { + return "", err + } + + // gets the info about the current user + authInfo := config.AuthInfos[config.Contexts[config.CurrentContext].AuthInfo] + + // gets the X509 certificate with current user credentials + var certs []*x509.Certificate + if len(authInfo.ClientCertificateData) > 0 { + // if the config file uses an embedded x509 certificate (e.g. kubelet.conf created by kubeadm), parse it + if certs, err = certutil.ParseCertsPEM(authInfo.ClientCertificateData); err != nil { + return "", err + } + } else if len(authInfo.ClientCertificate) > 0 { + // if the config file links an external x509 certificate (e.g. kubelet.conf created by TLS bootstrap), load it + if certs, err = certutil.CertsFromFile(authInfo.ClientCertificate); err != nil { + return "", err + } + } else { + return "", errors.New("Invalid kubelet.conf. X509 certificate expected") + } + + // We are only putting one certificate in the certificate pem file, so it's safe to just pick the first one + // TODO: Support multiple certs here in order to be able to rotate certs + cert := certs[0] + + // gets the node name from the certificate common name + return strings.TrimPrefix(cert.Subject.CommonName, constants.NodesUserPrefix), nil +} + +// getAPIEndpoint returns the APIEndpoint for the current node +func getAPIEndpoint(data map[string]string, nodeName string, apiEndpoint *kubeadmapi.APIEndpoint) error { + // gets the ClusterStatus from kubeadm-config + clusterStatusData, ok := data[constants.ClusterStatusConfigMapKey] + if !ok { + return fmt.Errorf("unexpected error when reading kubeadm-config ConfigMap: %s key value pair missing", constants.ClusterStatusConfigMapKey) + } + clusterStatus := &kubeadmapi.ClusterStatus{} + if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), []byte(clusterStatusData), clusterStatus); err != nil { + return err + } + + // gets the APIEndpoint for the current machine from the ClusterStatus + e, ok := clusterStatus.APIEndpoints[nodeName] + if !ok { + return errors.New("failed to get APIEndpoint information for this node") + } + + apiEndpoint.AdvertiseAddress = e.AdvertiseAddress + apiEndpoint.BindPort = e.BindPort + return nil +} + +// getComponentConfigs gets the component configs from the corresponding config maps +func getComponentConfigs(client clientset.Interface, clusterConfiguration *kubeadmapi.ClusterConfiguration) error { + // some config maps is versioned, so we need the KubernetesVersion for getting the right config map + k8sVersion := version.MustParseGeneric(clusterConfiguration.KubernetesVersion) + for kind, registration := range componentconfigs.Known { + obj, err := registration.GetFromConfigMap(client, k8sVersion) + if err != nil { + return err + } + + if ok := registration.SetToInternalConfig(obj, clusterConfiguration); !ok { + return fmt.Errorf("couldn't save componentconfig value for kind %q", string(kind)) + } + } + return nil } diff --git a/cmd/kubeadm/app/util/config/cluster_test.go b/cmd/kubeadm/app/util/config/cluster_test.go index 15abce8c44b29..77ba145f9171f 100644 --- a/cmd/kubeadm/app/util/config/cluster_test.go +++ b/cmd/kubeadm/app/util/config/cluster_test.go @@ -17,7 +17,10 @@ limitations under the License. package config import ( + "bytes" + "io/ioutil" "os" + "path/filepath" "strings" "testing" @@ -25,176 +28,657 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" clientsetfake "k8s.io/client-go/kubernetes/fake" - "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" + "k8s.io/kubernetes/pkg/util/version" ) -func TestFetchConfigFromFileOrCluster(t *testing.T) { +var k8sVersionString = "v1.12.0" +var k8sVersion = version.MustParseGeneric(k8sVersionString) +var nodeName = "mynode" +var cfgFiles = map[string][]byte{ + "MasterConfiguration_v1alpha2": []byte(` +apiVersion: kubeadm.k8s.io/v1alpha2 +kind: MasterConfiguration +kubernetesVersion: ` + k8sVersionString + ` +api: + advertiseAddress: 1.2.3.4 + bindPort: 1234 +`), + "InitConfiguration_v1alpha3": []byte(` +apiVersion: kubeadm.k8s.io/v1alpha3 +kind: InitConfiguration +`), + "ClusterConfiguration_v1alpha3": []byte(` +apiVersion: kubeadm.k8s.io/v1alpha3 +kind: ClusterConfiguration +kubernetesVersion: ` + k8sVersionString + ` +`), + "ClusterStatus_v1alpha3": []byte(` +apiVersion: kubeadm.k8s.io/v1alpha3 +kind: ClusterStatus +apiEndpoints: + ` + nodeName + `: + advertiseAddress: 1.2.3.4 + bindPort: 1234 +`), + "ClusterStatus_v1alpha3_Without_APIEndpoints": []byte(` +apiVersion: kubeadm.k8s.io/v1alpha3 +kind: ClusterStatus +`), + "Kube-proxy_componentconfig": []byte(` +apiVersion: kubeproxy.config.k8s.io/v1alpha1 +kind: KubeProxyConfiguration +`), + "Kubelet_componentconfig": []byte(` +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +`), +} + +var kubeletConfFiles = map[string][]byte{ + "withoutX509Cert": []byte(` +apiVersion: v1 +clusters: +- cluster: + server: https://10.0.2.15:6443 + name: kubernetes +contexts: +- context: + cluster: kubernetes + user: system:node:mynode + name: system:node:mynode@kubernetes +current-context: system:node:mynode@kubernetes +kind: Config +preferences: {} +users: +- name: system:node:mynode + user: +`), + "configWithEmbeddedCert": []byte(` +apiVersion: v1 +clusters: +- cluster: + server: https://10.0.2.15:6443 + name: kubernetes +contexts: +- context: + cluster: kubernetes + user: system:node:mynode + name: system:node:mynode@kubernetes +current-context: system:node:mynode@kubernetes +kind: Config +preferences: {} +users: +- name: system:node:mynode + user: + client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM4akNDQWRxZ0F3SUJBZ0lJQWl3VURhYk5vZ1F3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB4T0RBNU1ERXhOVE14TWpaYUZ3MHhPVEE1TURFeE5qQXhOVGxhTURReApGVEFUQmdOVkJBb1RESE41YzNSbGJUcHViMlJsY3pFYk1Ca0dBMVVFQXhNU2MzbHpkR1Z0T201dlpHVTZiWGx1CmIyUmxNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQWs2UXUzeStyNEZYUzZ4VkoKWU1vNE9kSkt3R1d1MHY4TEJIUnhhOUhvVHo1RXZLQnB1OVJoemt5dStUaFczb0xta2ZTRmNJcitHa0M5MW0zOApFelRmVE5JY2dsL0V5YkpmR1QvdGdUazZYd1kxY1UrUUdmSEFNNTBCVzFXTFVHc25CSllJZjA5eENnZTVoTkxLCnREeUJOWWNQZzg1bUJpOU9CNFJ2UlgyQVFRMjJwZ0xrQUpJWklOU0FEdUFrODN2b051SXM2YVY2bHBkR2Vva3YKdDlpTFdNR3p3a3ZTZUZQTlNGeWZ3Q055eENjb1FDQUNtSnJRS3NoeUE2bWNzdVhORWVXdlRQdVVFSWZPVFl4dwpxdkszRVBOK0xUYlA2anhUMWtTcFJUOSt4Z29uSlFhU0RsbUNBd20zRGJkSVppWUt3R2ppMkxKL0kvYWc0cTlzCjNLb0J2UUlEQVFBQm95Y3dKVEFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUgKQXdJd0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFLcVVrU21jdW85OG5EK015b005VFdEV0pyTndySXpQTUNqRQpCSkdyREhVaHIwcEZlRjc0RHViODNzRXlaNjFxNUVQd2Y0enNLSzdzdDRUTzZhcE9pZWJYVmN3dnZoa09HQ2dFCmFVdGNOMjFhUGxtU0tOd0c4ai8yK3ZhbU80bGplK1NnZzRUUVB0eDZWejh5VXN2RFhxSUZycjNNd1gzSDA1RW4KWXAzN05JYkhKbGxHUW5LVHA5aTg5aXF4WXVhSERqZldiVHlEY3B5NldNVjdVaFYvY1plc3lGL0NBamNHd1V6YgowRlo5bW5tMnFONlBGWHZ4RmdMSGFWZzN2SVVCbkNmVVVyY1BDNE94VFNPK21aUmUxazh3eUFpVWovSk0rZllvCkcrMi9sbThUYVZqb1U3Rmk1S2E1RzVIWTJHTGFSN1ArSXhZY3JNSENsNjJZN1JxY3JuYz0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= +`), + "configWithLinkedCert": []byte(` +apiVersion: v1 +clusters: +- cluster: + server: https://10.0.2.15:6443 + name: kubernetes +contexts: +- context: + cluster: kubernetes + user: system:node:mynode + name: system:node:mynode@kubernetes +current-context: system:node:mynode@kubernetes +kind: Config +preferences: {} +users: +- name: system:node:mynode + user: + client-certificate: kubelet.pem +`), +} + +var pemFiles = map[string][]byte{ + "mynode.pem": []byte(` +-----BEGIN CERTIFICATE----- +MIIC8jCCAdqgAwIBAgIIAiwUDabNogQwDQYJKoZIhvcNAQELBQAwFTETMBEGA1UE +AxMKa3ViZXJuZXRlczAeFw0xODA5MDExNTMxMjZaFw0xOTA5MDExNjAxNTlaMDQx +FTATBgNVBAoTDHN5c3RlbTpub2RlczEbMBkGA1UEAxMSc3lzdGVtOm5vZGU6bXlu +b2RlMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAk6Qu3y+r4FXS6xVJ +YMo4OdJKwGWu0v8LBHRxa9HoTz5EvKBpu9Rhzkyu+ThW3oLmkfSFcIr+GkC91m38 +EzTfTNIcgl/EybJfGT/tgTk6XwY1cU+QGfHAM50BW1WLUGsnBJYIf09xCge5hNLK +tDyBNYcPg85mBi9OB4RvRX2AQQ22pgLkAJIZINSADuAk83voNuIs6aV6lpdGeokv +t9iLWMGzwkvSeFPNSFyfwCNyxCcoQCACmJrQKshyA6mcsuXNEeWvTPuUEIfOTYxw +qvK3EPN+LTbP6jxT1kSpRT9+xgonJQaSDlmCAwm3DbdIZiYKwGji2LJ/I/ag4q9s +3KoBvQIDAQABoycwJTAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUH +AwIwDQYJKoZIhvcNAQELBQADggEBAKqUkSmcuo98nD+MyoM9TWDWJrNwrIzPMCjE +BJGrDHUhr0pFeF74Dub83sEyZ61q5EPwf4zsKK7st4TO6apOiebXVcwvvhkOGCgE +aUtcN21aPlmSKNwG8j/2+vamO4lje+Sgg4TQPtx6Vz8yUsvDXqIFrr3MwX3H05En +Yp37NIbHJllGQnKTp9i89iqxYuaHDjfWbTyDcpy6WMV7UhV/cZesyF/CAjcGwUzb +0FZ9mnm2qN6PFXvxFgLHaVg3vIUBnCfUUrcPC4OxTSO+mZRe1k8wyAiUj/JM+fYo +G+2/lm8TaVjoU7Fi5Ka5G5HY2GLaR7P+IxYcrMHCl62Y7Rqcrnc= +-----END CERTIFICATE----- +`), +} + +func TestLoadInitConfigurationFromFile(t *testing.T) { + // Create temp folder for the test case + tmpdir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("Couldn't create tmpdir") + } + defer os.RemoveAll(tmpdir) + var tests = []struct { - name string - cfgPath string - testCfg *kubeadmapi.InitConfiguration - expectErr string + name string + fileContents []byte }{ { - name: "fetch valid config from configMap", - testCfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{ - AdvertiseAddress: "1.2.3.4", - BindPort: 6443, - }, - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - KubernetesVersion: "v1.10.3", - Etcd: kubeadm.Etcd{ - Local: &kubeadm.LocalEtcd{ - DataDir: "/some/path", - }, - }, - Networking: kubeadm.Networking{ - ServiceSubnet: "10.96.0.1/12", - DNSDomain: "cluster.local", - PodSubnet: "10.0.1.15/16", - }, - CertificatesDir: "/some/other/cert/dir", - }, - BootstrapTokens: []kubeadm.BootstrapToken{ - { - Token: &kubeadm.BootstrapTokenString{ - ID: "abcdef", - Secret: "abcdef0123456789", - }, + name: "v1alpha2.MasterConfiguration", + fileContents: cfgFiles["MasterConfiguration_v1alpha2"], + }, + { + name: "v1alpha3.partial1", + fileContents: cfgFiles["InitConfiguration_v1alpha3"], + }, + { + name: "v1alpha3.partial2", + fileContents: cfgFiles["ClusterConfiguration_v1alpha3"], + }, + { + name: "v1alpha3.full", + fileContents: bytes.Join([][]byte{ + cfgFiles["InitConfiguration_v1alpha3"], + cfgFiles["ClusterConfiguration_v1alpha3"], + cfgFiles["Kube-proxy_componentconfig"], + cfgFiles["Kubelet_componentconfig"], + }, []byte(kubeadmconstants.YAMLDocumentSeparator)), + }, + } + + for _, rt := range tests { + t.Run(rt.name, func(t2 *testing.T) { + cfgPath := filepath.Join(tmpdir, rt.name) + err := ioutil.WriteFile(cfgPath, rt.fileContents, 0644) + if err != nil { + t.Errorf("Couldn't create file") + return + } + + obj, err := loadInitConfigurationFromFile(cfgPath) + if err != nil { + t.Errorf("Error reading file: %v", err) + return + } + + if obj == nil { + t.Errorf("Unexpected nil return value") + } + }) + } +} + +func TestGetNodeNameFromKubeletConfig(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("Couldn't create tmpdir") + } + defer os.RemoveAll(tmpdir) + + var tests = []struct { + name string + kubeconfigContent []byte + pemContent []byte + expectedError bool + }{ + { + name: "valid - with embedded cert", + kubeconfigContent: kubeletConfFiles["configWithEmbeddedCert"], + }, + { + name: "invalid - linked cert missing", + kubeconfigContent: kubeletConfFiles["configWithLinkedCert"], + expectedError: true, + }, + { + name: "valid - with linked cert", + kubeconfigContent: kubeletConfFiles["configWithLinkedCert"], + pemContent: pemFiles["mynode.pem"], + }, + { + name: "invalid - without embedded or linked X509Cert", + kubeconfigContent: kubeletConfFiles["withoutX509Cert"], + expectedError: true, + }, + } + + for _, rt := range tests { + t.Run(rt.name, func(t2 *testing.T) { + if len(rt.pemContent) > 0 { + pemPath := filepath.Join(tmpdir, "kubelet.pem") + err := ioutil.WriteFile(pemPath, rt.pemContent, 0644) + if err != nil { + t.Errorf("Couldn't create pem file: %v", err) + return + } + rt.kubeconfigContent = []byte(strings.Replace(string(rt.kubeconfigContent), "kubelet.pem", pemPath, -1)) + } + + kubeconfigPath := filepath.Join(tmpdir, kubeadmconstants.KubeletKubeConfigFileName) + err := ioutil.WriteFile(kubeconfigPath, rt.kubeconfigContent, 0644) + if err != nil { + t.Errorf("Couldn't create kubeconfig: %v", err) + return + } + + name, err := getNodeNameFromKubeletConfig(tmpdir) + if rt.expectedError != (err != nil) { + t.Errorf("unexpected return err from getNodeRegistration: %v", err) + return + } + if rt.expectedError { + return + } + + if name != nodeName { + t.Errorf("invalid name") + } + }) + } +} + +func TestGetNodeRegistration(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("Couldn't create tmpdir") + } + defer os.RemoveAll(tmpdir) + + var tests = []struct { + name string + fileContents []byte + node *v1.Node + expectedError bool + }{ + { + name: "invalid - no kubelet.conf", + expectedError: true, + }, + { + name: "valid", + fileContents: kubeletConfFiles["configWithEmbeddedCert"], + node: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + Annotations: map[string]string{ + kubeadmconstants.AnnotationKubeadmCRISocket: "myCRIsocket", }, }, - NodeRegistration: kubeadm.NodeRegistrationOptions{ - Name: "node-foo", - CRISocket: "/var/run/custom-cri.sock", + Spec: v1.NodeSpec{ + Taints: []v1.Taint{kubeadmconstants.MasterTaint}, }, }, }, { - name: "fetch invalid config from configMap", - testCfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{ - AdvertiseAddress: "1.2.3.4", - BindPort: 6443, + name: "invalid - no node", + fileContents: kubeletConfFiles["configWithEmbeddedCert"], + expectedError: true, + }, + } + + for _, rt := range tests { + t.Run(rt.name, func(t2 *testing.T) { + cfgPath := filepath.Join(tmpdir, kubeadmconstants.KubeletKubeConfigFileName) + if len(rt.fileContents) > 0 { + err := ioutil.WriteFile(cfgPath, rt.fileContents, 0644) + if err != nil { + t.Errorf("Couldn't create file") + return + } + } + + client := clientsetfake.NewSimpleClientset() + + if rt.node != nil { + _, err := client.CoreV1().Nodes().Create(rt.node) + if err != nil { + t.Errorf("couldn't create Node") + return + } + } + + cfg := &kubeadmapi.InitConfiguration{} + err = getNodeRegistration(tmpdir, client, &cfg.NodeRegistration) + if rt.expectedError != (err != nil) { + t.Errorf("unexpected return err from getNodeRegistration: %v", err) + return + } + if rt.expectedError { + return + } + + if cfg.NodeRegistration.Name != nodeName { + t.Errorf("invalid cfg.NodeRegistration.Name") + } + if cfg.NodeRegistration.CRISocket != "myCRIsocket" { + t.Errorf("invalid cfg.NodeRegistration.CRISocket") + } + if len(cfg.NodeRegistration.Taints) != 1 { + t.Errorf("invalid cfg.NodeRegistration.Taints") + } + }) + } +} + +func TestGetAPIEndpoint(t *testing.T) { + var tests = []struct { + name string + configMap fakeConfigMap + expectedError bool + }{ + { + name: "valid", + configMap: fakeConfigMap{ + name: kubeadmconstants.InitConfigurationConfigMap, // ClusterConfiguration from kubeadm-config. + data: map[string]string{ + kubeadmconstants.ClusterStatusConfigMapKey: string(cfgFiles["ClusterStatus_v1alpha3"]), }, - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - KubernetesVersion: "v1.10.3", - Etcd: kubeadm.Etcd{ - Local: &kubeadm.LocalEtcd{ - DataDir: "/some/path", - }, - }, - Networking: kubeadm.Networking{ - ServiceSubnet: "10.96.0.1/12", - DNSDomain: "cluster.local", - PodSubnet: "10.0.1.15", // wrong + }, + }, + { + name: "invalid - No CLusterStatus in kubeadm-config ConfigMap", + configMap: fakeConfigMap{ + name: kubeadmconstants.InitConfigurationConfigMap, // ClusterConfiguration from kubeadm-config. + data: map[string]string{}, + }, + expectedError: true, + }, + { + name: "invalid - CLusterStatus without APIEndopoints", + configMap: fakeConfigMap{ + name: kubeadmconstants.InitConfigurationConfigMap, // ClusterConfiguration from kubeadm-config. + data: map[string]string{ + kubeadmconstants.ClusterStatusConfigMapKey: string(cfgFiles["ClusterStatus_v1alpha3_Without_APIEndpoints"]), + }, + }, + expectedError: true, + }, + } + + for _, rt := range tests { + t.Run(rt.name, func(t *testing.T) { + cfg := &kubeadmapi.InitConfiguration{} + err := getAPIEndpoint(rt.configMap.data, nodeName, &cfg.APIEndpoint) + if rt.expectedError != (err != nil) { + t.Errorf("unexpected return err from getInitConfigurationFromCluster: %v", err) + return + } + if rt.expectedError { + return + } + + if cfg.APIEndpoint.AdvertiseAddress != "1.2.3.4" || cfg.APIEndpoint.BindPort != 1234 { + t.Errorf("invalid cfg.APIEndpoint") + } + }) + } +} + +func TestGetComponentConfigs(t *testing.T) { + var tests = []struct { + name string + configMaps []fakeConfigMap + expectedError bool + }{ + { + name: "valid", + configMaps: []fakeConfigMap{ + { + name: kubeadmconstants.KubeProxyConfigMap, // Kube-proxy component config from corresponding ConfigMap. + data: map[string]string{ + kubeadmconstants.KubeProxyConfigMapKey: string(cfgFiles["Kube-proxy_componentconfig"]), }, - CertificatesDir: "/some/other/cert/dir", - }, - BootstrapTokens: []kubeadm.BootstrapToken{ - { - Token: &kubeadm.BootstrapTokenString{ - ID: "abcdef", - Secret: "abcdef0123456789", - }, + }, + { + name: kubeadmconstants.GetKubeletConfigMapName(k8sVersion), // Kubelet component config from corresponding ConfigMap. + data: map[string]string{ + kubeadmconstants.KubeletBaseConfigurationConfigMapKey: string(cfgFiles["Kubelet_componentconfig"]), }, }, - NodeRegistration: kubeadm.NodeRegistrationOptions{ - Name: "node-foo", - CRISocket: "/var/run/custom-cri.sock", + }, + }, + { + name: "invalid - No kubelet component config ConfigMap", + configMaps: []fakeConfigMap{ + { + name: kubeadmconstants.KubeProxyConfigMap, + data: map[string]string{ + kubeadmconstants.KubeProxyConfigMapKey: string(cfgFiles["Kube-proxy_componentconfig"]), + }, }, }, - expectErr: "couldn't parse subnet", + expectedError: true, }, { - name: "fetch valid config from cfgPath", - cfgPath: "testdata/conversion/master/v1alpha3.yaml", - testCfg: &kubeadmapi.InitConfiguration{ - APIEndpoint: kubeadmapi.APIEndpoint{ - AdvertiseAddress: "1.2.3.4", - BindPort: 6443, + name: "invalid - No kube-proxy component config ConfigMap", + configMaps: []fakeConfigMap{ + { + name: kubeadmconstants.GetKubeletConfigMapName(k8sVersion), + data: map[string]string{ + kubeadmconstants.KubeletBaseConfigurationConfigMapKey: string(cfgFiles["Kubelet_componentconfig"]), + }, }, + }, + expectedError: true, + }, + } + + for _, rt := range tests { + t.Run(rt.name, func(t *testing.T) { + client := clientsetfake.NewSimpleClientset() + + for _, c := range rt.configMaps { + err := c.create(client) + if err != nil { + t.Errorf("couldn't create ConfigMap %s", c.name) + return + } + } + + cfg := &kubeadmapi.InitConfiguration{ ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - KubernetesVersion: "v1.10.3", - Etcd: kubeadm.Etcd{ - Local: &kubeadm.LocalEtcd{ - DataDir: "/some/path", - }, - }, - Networking: kubeadm.Networking{ - ServiceSubnet: "10.96.0.1/12", - DNSDomain: "cluster.local", - PodSubnet: "10.0.1.15", - }, - CertificatesDir: "/some/other/cert/dir", - }, - BootstrapTokens: []kubeadm.BootstrapToken{ - { - Token: &kubeadm.BootstrapTokenString{ - ID: "abcdef", - Secret: "abcdef0123456789", - }, - }, + KubernetesVersion: k8sVersionString, }, - NodeRegistration: kubeadm.NodeRegistrationOptions{ - Name: "node-foo", - CRISocket: "/var/run/custom-cri.sock", + } + err := getComponentConfigs(client, &cfg.ClusterConfiguration) + if rt.expectedError != (err != nil) { + t.Errorf("unexpected return err from getInitConfigurationFromCluster: %v", err) + return + } + if rt.expectedError { + return + } + + // Test expected values in InitConfiguration + if cfg.ComponentConfigs.Kubelet == nil { + t.Errorf("invalid cfg.ComponentConfigs.Kubelet") + } + if cfg.ComponentConfigs.KubeProxy == nil { + t.Errorf("invalid cfg.ComponentConfigs.KubeProxy") + } + }) + } +} + +func TestGetInitConfigurationFromCluster(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("Couldn't create tmpdir") + } + defer os.RemoveAll(tmpdir) + + var tests = []struct { + name string + fileContents []byte + node *v1.Node + configMaps []fakeConfigMap + newControlPlane bool + expectedError bool + }{ + { //TODO: remove in V1.13 + name: "before v1.11", // single YAML, with a v1alpha2.MasterConfiguration object (with embedded component configs) + configMaps: []fakeConfigMap{ + { + name: kubeadmconstants.InitConfigurationConfigMap, + data: map[string]string{ + kubeadmconstants.InitConfigurationConfigMapKey: string(cfgFiles["MasterConfiguration_v1alpha2"]), + }, }, }, }, { - name: "fetch invalid config from cfgPath", - cfgPath: "testdata/validation/invalid_mastercfg.yaml", - expectErr: "was not of the form", + name: "invalid - No kubeadm-config ConfigMap", + expectedError: true, + }, + { + name: "invalid - No CLusterConfiguration in kubeadm-config ConfigMap", + configMaps: []fakeConfigMap{ + { + name: kubeadmconstants.InitConfigurationConfigMap, // ClusterConfiguration from kubeadm-config. + data: map[string]string{}, + }, + }, + expectedError: true, }, { - name: "fetch config from not exist cfgPath", - cfgPath: "doesnotexist.yaml", - expectErr: "no such file or directory", + name: "valid - new control plane == false", // InitConfiguration composed with data from different places, with also node specific information from ClusterStatus and node + configMaps: []fakeConfigMap{ + { + name: kubeadmconstants.InitConfigurationConfigMap, // ClusterConfiguration from kubeadm-config. + data: map[string]string{ + kubeadmconstants.ClusterConfigurationConfigMapKey: string(cfgFiles["ClusterConfiguration_v1alpha3"]), + kubeadmconstants.ClusterStatusConfigMapKey: string(cfgFiles["ClusterStatus_v1alpha3"]), + }, + }, + { + name: kubeadmconstants.KubeProxyConfigMap, // Kube-proxy component config from corresponding ConfigMap. + data: map[string]string{ + kubeadmconstants.KubeProxyConfigMapKey: string(cfgFiles["Kube-proxy_componentconfig"]), + }, + }, + { + name: kubeadmconstants.GetKubeletConfigMapName(k8sVersion), // Kubelet component config from corresponding ConfigMap. + data: map[string]string{ + kubeadmconstants.KubeletBaseConfigurationConfigMapKey: string(cfgFiles["Kubelet_componentconfig"]), + }, + }, + }, + fileContents: kubeletConfFiles["configWithEmbeddedCert"], + node: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + Annotations: map[string]string{ + kubeadmconstants.AnnotationKubeadmCRISocket: "myCRIsocket", + }, + }, + Spec: v1.NodeSpec{ + Taints: []v1.Taint{kubeadmconstants.MasterTaint}, + }, + }, }, { - name: "fetch config when no configMap and no cfgPath", - expectErr: "not found", + name: "valid - new control plane == true", // InitConfiguration composed with data from different places, without node specific information + configMaps: []fakeConfigMap{ + { + name: kubeadmconstants.InitConfigurationConfigMap, // ClusterConfiguration from kubeadm-config. + data: map[string]string{ + kubeadmconstants.ClusterConfigurationConfigMapKey: string(cfgFiles["ClusterConfiguration_v1alpha3"]), + }, + }, + { + name: kubeadmconstants.KubeProxyConfigMap, // Kube-proxy component config from corresponding ConfigMap. + data: map[string]string{ + kubeadmconstants.KubeProxyConfigMapKey: string(cfgFiles["Kube-proxy_componentconfig"]), + }, + }, + { + name: kubeadmconstants.GetKubeletConfigMapName(k8sVersion), // Kubelet component config from corresponding ConfigMap. + data: map[string]string{ + kubeadmconstants.KubeletBaseConfigurationConfigMapKey: string(cfgFiles["Kubelet_componentconfig"]), + }, + }, + }, + newControlPlane: true, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { + for _, rt := range tests { + t.Run(rt.name, func(t *testing.T) { + cfgPath := filepath.Join(tmpdir, kubeadmconstants.KubeletKubeConfigFileName) + if len(rt.fileContents) > 0 { + err := ioutil.WriteFile(cfgPath, rt.fileContents, 0644) + if err != nil { + t.Errorf("Couldn't create file") + return + } + } + client := clientsetfake.NewSimpleClientset() - if tt.testCfg != nil { - if err := createConfigMapWithCfg(tt.testCfg, client); err != nil { - t.Errorf("UploadConfiguration failed err: %v", err) + + if rt.node != nil { + _, err := client.CoreV1().Nodes().Create(rt.node) + if err != nil { + t.Errorf("couldn't create Node") + return } } - _, err := FetchConfigFromFileOrCluster(client, os.Stdout, "upgrade/config", tt.cfgPath) - if err != nil { - if len(tt.expectErr) == 0 { - t.Fatalf("expected no err, but got err: %v", err) - } else if !strings.Contains(err.Error(), tt.expectErr) { - t.Errorf("expected contain err: %v, but got err: %v", tt.expectErr, err) + + for _, c := range rt.configMaps { + err := c.create(client) + if err != nil { + t.Errorf("couldn't create ConfigMap %s", c.name) + return } } + + cfg, err := getInitConfigurationFromCluster(tmpdir, client, rt.newControlPlane) + if rt.expectedError != (err != nil) { + t.Errorf("unexpected return err from getInitConfigurationFromCluster: %v", err) + return + } + if rt.expectedError { + return + } + + // Test expected values in InitConfiguration + if cfg == nil { + t.Errorf("unexpected nil return value") + } + if cfg.ClusterConfiguration.KubernetesVersion != k8sVersionString { + t.Errorf("invalid ClusterConfiguration.KubernetesVersion") + } + if !rt.newControlPlane && (cfg.APIEndpoint.AdvertiseAddress != "1.2.3.4" || cfg.APIEndpoint.BindPort != 1234) { + t.Errorf("invalid cfg.APIEndpoint") + } + if cfg.ComponentConfigs.Kubelet == nil { + t.Errorf("invalid cfg.ComponentConfigs.Kubelet") + } + if cfg.ComponentConfigs.KubeProxy == nil { + t.Errorf("invalid cfg.ComponentConfigs.KubeProxy") + } }) } } -// createConfigMapWithCfg create a ConfigMap with InitConfiguration for TestFetchConfigFromFileOrCluster -func createConfigMapWithCfg(cfgToCreate *kubeadmapi.InitConfiguration, client clientset.Interface) error { - cfgYaml, err := MarshalKubeadmConfigObject(cfgToCreate) - if err != nil { - return err - } +type fakeConfigMap struct { + name string + data map[string]string +} +func (c *fakeConfigMap) create(client clientset.Interface) error { return apiclient.CreateOrUpdateConfigMap(client, &v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: kubeadmconstants.InitConfigurationConfigMap, + Name: c.name, Namespace: metav1.NamespaceSystem, }, - Data: map[string]string{ - kubeadmconstants.InitConfigurationConfigMapKey: string(cfgYaml), - }, + Data: c.data, }) } diff --git a/cmd/kubeadm/app/util/config/masterconfig.go b/cmd/kubeadm/app/util/config/masterconfig.go index d3fde8264c927..c0d7f48badf30 100644 --- a/cmd/kubeadm/app/util/config/masterconfig.go +++ b/cmd/kubeadm/app/util/config/masterconfig.go @@ -23,6 +23,7 @@ import ( "net" "reflect" "sort" + "strconv" "github.com/golang/glog" @@ -52,7 +53,7 @@ func SetInitDynamicDefaults(cfg *kubeadmapi.InitConfiguration) error { if err := SetAPIEndpointDynamicDefaults(&cfg.APIEndpoint); err != nil { return err } - if err := SetClusterDynamicDefaults(&cfg.ClusterConfiguration, cfg.APIEndpoint.AdvertiseAddress); err != nil { + if err := SetClusterDynamicDefaults(&cfg.ClusterConfiguration, cfg.APIEndpoint.AdvertiseAddress, cfg.APIEndpoint.BindPort); err != nil { return err } return nil @@ -119,7 +120,7 @@ func SetAPIEndpointDynamicDefaults(cfg *kubeadmapi.APIEndpoint) error { } // SetClusterDynamicDefaults checks and sets configuration values for the InitConfiguration object -func SetClusterDynamicDefaults(cfg *kubeadmapi.ClusterConfiguration, advertiseAddress string) error { +func SetClusterDynamicDefaults(cfg *kubeadmapi.ClusterConfiguration, advertiseAddress string, bindPort int32) error { // Default all the embedded ComponentConfig structs componentconfigs.Known.Default(cfg) @@ -135,6 +136,19 @@ func SetClusterDynamicDefaults(cfg *kubeadmapi.ClusterConfiguration, advertiseAd return err } + // If ControlPlaneEndpoint is specified without a port number defaults it to + // the bindPort number of the APIEndpoint. + // This will allow join of additional control plane instances with different bindPort number + if cfg.ControlPlaneEndpoint != "" { + host, port, err := kubeadmutil.ParseHostPort(cfg.ControlPlaneEndpoint) + if err != nil { + return err + } + if port == "" { + cfg.ControlPlaneEndpoint = net.JoinHostPort(host, strconv.FormatInt(int64(bindPort), 10)) + } + } + // Downcase SANs. Some domain names (like ELBs) have capitals in them. LowercaseSANs(cfg.APIServerCertSANs) return nil diff --git a/cmd/kubeadm/app/util/endpoint.go b/cmd/kubeadm/app/util/endpoint.go index 8f8fed071c588..a092e84d3d054 100644 --- a/cmd/kubeadm/app/util/endpoint.go +++ b/cmd/kubeadm/app/util/endpoint.go @@ -60,7 +60,9 @@ func GetMasterEndpoint(cfg *kubeadmapi.InitConfiguration) (string, error) { // if a port is provided within the controlPlaneAddress warn the users we are using it, else use the bindport if port != "" { - fmt.Println("[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address") + if port != bindPortString { + fmt.Println("[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address") + } } else { port = bindPortString }