diff --git a/kubernetes/crds/machine.sapcloud.io_machineclasses.yaml b/kubernetes/crds/machine.sapcloud.io_machineclasses.yaml index 29868892b..01cd76fae 100644 --- a/kubernetes/crds/machine.sapcloud.io_machineclasses.yaml +++ b/kubernetes/crds/machine.sapcloud.io_machineclasses.yaml @@ -25,6 +25,21 @@ spec: of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string + credentialsSecretRef: + description: CredentialsSecretRef can optionally store the credentials (in + this case the SecretRef does not need to store them). This might be useful + if multiple machine classes with the same credentials but different user-datas + are used. + properties: + name: + description: Name is unique within a namespace to reference a secret + resource. + type: string + namespace: + description: Namespace defines the space within which the secret name + must be unique. + type: string + type: object kind: description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client @@ -40,8 +55,8 @@ spec: description: Provider-specific configuration to use during node creation. type: object secretRef: - description: SecretRef stores the necessary secrets such as credetials or - userdata. + description: SecretRef stores the necessary secrets such as credentials + or userdata. properties: name: description: Name is unique within a namespace to reference a secret diff --git a/pkg/apis/machine/types.go b/pkg/apis/machine/types.go index 6fe524ba0..fb0e5d3e8 100644 --- a/pkg/apis/machine/types.go +++ b/pkg/apis/machine/types.go @@ -1206,8 +1206,11 @@ type MachineClass struct { metav1.ObjectMeta // Provider-specific configuration to use during node creation. ProviderSpec runtime.RawExtension - // SecretRef stores the necessary secrets such as credetials or userdata. + // SecretRef stores the necessary secrets such as credentials or userdata. SecretRef *corev1.SecretReference + // CredentialsSecretRef can optionally store the credentials (in this case the SecretRef does not need to store them). + // This might be useful if multiple machine classes with the same credentials but different user-datas are used. + CredentialsSecretRef *corev1.SecretReference // Provider is the combination of name and location of cloud-specific drivers. // eg. awsdriver//127.0.0.1:8080 Provider string diff --git a/pkg/apis/machine/v1alpha1/machineclass_types.go b/pkg/apis/machine/v1alpha1/machineclass_types.go index b26f0d2fb..905835ad7 100644 --- a/pkg/apis/machine/v1alpha1/machineclass_types.go +++ b/pkg/apis/machine/v1alpha1/machineclass_types.go @@ -39,8 +39,11 @@ type MachineClass struct { metav1.ObjectMeta `json:"metadata,omitempty"` // Provider-specific configuration to use during node creation. ProviderSpec runtime.RawExtension `json:"providerSpec"` - // SecretRef stores the necessary secrets such as credetials or userdata. + // SecretRef stores the necessary secrets such as credentials or userdata. SecretRef *corev1.SecretReference `json:"secretRef,omitempty"` + // CredentialsSecretRef can optionally store the credentials (in this case the SecretRef does not need to store them). + // This might be useful if multiple machine classes with the same credentials but different user-datas are used. + CredentialsSecretRef *corev1.SecretReference `json:"credentialsSecretRef,omitempty"` // Provider is the combination of name and location of cloud-specific drivers. Provider string `json:"provider,omitempty"` } diff --git a/pkg/apis/machine/v1alpha1/zz_generated.conversion.go b/pkg/apis/machine/v1alpha1/zz_generated.conversion.go index f78cf0c5b..5d209710a 100644 --- a/pkg/apis/machine/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/machine/v1alpha1/zz_generated.conversion.go @@ -2041,6 +2041,7 @@ func autoConvert_v1alpha1_MachineClass_To_machine_MachineClass(in *MachineClass, out.ObjectMeta = in.ObjectMeta out.ProviderSpec = in.ProviderSpec out.SecretRef = (*v1.SecretReference)(unsafe.Pointer(in.SecretRef)) + out.CredentialsSecretRef = (*v1.SecretReference)(unsafe.Pointer(in.CredentialsSecretRef)) out.Provider = in.Provider return nil } @@ -2054,6 +2055,7 @@ func autoConvert_machine_MachineClass_To_v1alpha1_MachineClass(in *machine.Machi out.ObjectMeta = in.ObjectMeta out.ProviderSpec = in.ProviderSpec out.SecretRef = (*v1.SecretReference)(unsafe.Pointer(in.SecretRef)) + out.CredentialsSecretRef = (*v1.SecretReference)(unsafe.Pointer(in.CredentialsSecretRef)) out.Provider = in.Provider return nil } diff --git a/pkg/apis/machine/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/machine/v1alpha1/zz_generated.deepcopy.go index 9ffe5cd56..1a4f9eee3 100644 --- a/pkg/apis/machine/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/machine/v1alpha1/zz_generated.deepcopy.go @@ -1157,6 +1157,11 @@ func (in *MachineClass) DeepCopyInto(out *MachineClass) { *out = new(v1.SecretReference) **out = **in } + if in.CredentialsSecretRef != nil { + in, out := &in.CredentialsSecretRef, &out.CredentialsSecretRef + *out = new(v1.SecretReference) + **out = **in + } return } diff --git a/pkg/apis/machine/zz_generated.deepcopy.go b/pkg/apis/machine/zz_generated.deepcopy.go index 124037b61..b475c974f 100644 --- a/pkg/apis/machine/zz_generated.deepcopy.go +++ b/pkg/apis/machine/zz_generated.deepcopy.go @@ -1157,6 +1157,11 @@ func (in *MachineClass) DeepCopyInto(out *MachineClass) { *out = new(v1.SecretReference) **out = **in } + if in.CredentialsSecretRef != nil { + in, out := &in.CredentialsSecretRef, &out.CredentialsSecretRef + *out = new(v1.SecretReference) + **out = **in + } return } diff --git a/pkg/openapi/openapi_generated.go b/pkg/openapi/openapi_generated.go index 4e5cd1af9..233292882 100644 --- a/pkg/openapi/openapi_generated.go +++ b/pkg/openapi/openapi_generated.go @@ -2305,7 +2305,13 @@ func schema_pkg_apis_machine_v1alpha1_MachineClass(ref common.ReferenceCallback) }, "secretRef": { SchemaProps: spec.SchemaProps{ - Description: "SecretRef stores the necessary secrets such as credetials or userdata.", + Description: "SecretRef stores the necessary secrets such as credentials or userdata.", + Ref: ref("k8s.io/api/core/v1.SecretReference"), + }, + }, + "credentialsSecretRef": { + SchemaProps: spec.SchemaProps{ + Description: "CredentialsSecretRef can optionally store the credentials (in this case the SecretRef does not need to store them). This might be useful if multiple machine classes with the same credentials but different user-datas are used.", Ref: ref("k8s.io/api/core/v1.SecretReference"), }, }, diff --git a/pkg/util/provider/machinecontroller/machine.go b/pkg/util/provider/machinecontroller/machine.go index 92318027a..14ac58def 100644 --- a/pkg/util/provider/machinecontroller/machine.go +++ b/pkg/util/provider/machinecontroller/machine.go @@ -134,7 +134,7 @@ func (c *controller) reconcileClusterMachine(machine *v1alpha1.Machine) (machine return machineutils.LongRetry, err } - machineClass, secret, retry, err := c.ValidateMachineClass(&machine.Spec.Class) + machineClass, secretData, retry, err := c.ValidateMachineClass(&machine.Spec.Class) if err != nil { klog.Error(err) return retry, err @@ -145,7 +145,7 @@ func (c *controller) reconcileClusterMachine(machine *v1alpha1.Machine) (machine return c.triggerDeletionFlow(&driver.DeleteMachineRequest{ Machine: machine, MachineClass: machineClass, - Secret: secret, + Secret: &corev1.Secret{Data: secretData}, }) } @@ -161,12 +161,11 @@ func (c *controller) reconcileClusterMachine(machine *v1alpha1.Machine) (machine return retry, err } } - if machine.Spec.ProviderID == "" || machine.Status.CurrentStatus.Phase == "" || machine.Status.Node == "" { return c.triggerCreationFlow(&driver.CreateMachineRequest{ Machine: machine, MachineClass: machineClass, - Secret: secret, + Secret: &corev1.Secret{Data: secretData}, }) } diff --git a/pkg/util/provider/machinecontroller/machine_safety.go b/pkg/util/provider/machinecontroller/machine_safety.go index 2e9637e8d..d71a20f0e 100644 --- a/pkg/util/provider/machinecontroller/machine_safety.go +++ b/pkg/util/provider/machinecontroller/machine_safety.go @@ -170,10 +170,7 @@ func (c *controller) checkMachineClasses() (machineutils.RetryPeriod, error) { } for _, machineClass := range MachineClasses { - retry, err := c.checkMachineClass( - machineClass, - machineClass.SecretRef, - ) + retry, err := c.checkMachineClass(machineClass) if err != nil { return retry, err } @@ -183,20 +180,18 @@ func (c *controller) checkMachineClasses() (machineutils.RetryPeriod, error) { } // checkMachineClass checks a particular machineClass for orphan instances -func (c *controller) checkMachineClass( - machineClass *v1alpha1.MachineClass, - secretRef *corev1.SecretReference) (machineutils.RetryPeriod, error) { - - // Get secret - secret, err := c.getSecret(secretRef, machineClass.Name) - if err != nil || secret == nil { - klog.Errorf("SafetyController: Secret reference not found for MachineClass: %q", machineClass.Name) +func (c *controller) checkMachineClass(machineClass *v1alpha1.MachineClass) (machineutils.RetryPeriod, error) { + + // Get secret data + secretData, err := c.getSecretData(machineClass.Name, machineClass.SecretRef, machineClass.CredentialsSecretRef) + if err != nil { + klog.Errorf("SafetyController: Secret Data could not be computed for MachineClass: %q", machineClass.Name) return machineutils.LongRetry, err } listMachineResponse, err := c.driver.ListMachines(context.TODO(), &driver.ListMachinesRequest{ MachineClass: machineClass, - Secret: secret, + Secret: &corev1.Secret{Data: secretData}, }) if err != nil { klog.Errorf("SafetyController: Failed to LIST VMs at provider. Error: %s", err) @@ -242,7 +237,7 @@ func (c *controller) checkMachineClass( _, err := c.driver.DeleteMachine(context.TODO(), &driver.DeleteMachineRequest{ Machine: machine, MachineClass: machineClass, - Secret: secret, + Secret: &corev1.Secret{Data: secretData}, }) if err != nil { klog.Errorf("SafetyController: Error while trying to DELETE VM on CP - %s. Shall retry in next safety controller sync.", err) diff --git a/pkg/util/provider/machinecontroller/machine_test.go b/pkg/util/provider/machinecontroller/machine_test.go index 1d6c56bbd..b7709c839 100644 --- a/pkg/util/provider/machinecontroller/machine_test.go +++ b/pkg/util/provider/machinecontroller/machine_test.go @@ -229,7 +229,7 @@ var _ = Describe("machine", func() { } type expect struct { machineClass interface{} - secret *corev1.Secret + secretData map[string][]byte err bool } type data struct { @@ -262,17 +262,17 @@ var _ = Describe("machine", func() { defer trackers.Stop() waitForCacheSync(stop, controller) - machineClass, secret, _, err := controller.ValidateMachineClass(data.action) + machineClass, secretData, _, err := controller.ValidateMachineClass(data.action) if data.expect.machineClass == nil { Expect(machineClass).To(BeNil()) } else { Expect(machineClass).To(Equal(data.expect.machineClass)) } - if data.expect.secret == nil { - Expect(secret).To(BeNil()) + if data.expect.secretData == nil { + Expect(secretData).To(BeNil()) } else { - Expect(secret).To(Equal(data.expect.secret)) + Expect(secretData).To(Equal(data.expect.secretData)) } if !data.expect.err { Expect(err).To(BeNil()) @@ -323,6 +323,7 @@ var _ = Describe("machine", func() { secrets: []*corev1.Secret{ { ObjectMeta: *newObjectMeta(objMeta, 0), + Data: map[string][]byte{"foo": []byte("bar")}, }, }, aws: []*v1alpha1.MachineClass{ @@ -341,10 +342,8 @@ var _ = Describe("machine", func() { ObjectMeta: *newObjectMeta(objMeta, 0), SecretRef: newSecretReference(objMeta, 0), }, - secret: &corev1.Secret{ - ObjectMeta: *newObjectMeta(objMeta, 0), - }, - err: false, + secretData: map[string][]byte{"foo": []byte("bar")}, + err: false, }, }), ) diff --git a/pkg/util/provider/machinecontroller/machine_util.go b/pkg/util/provider/machinecontroller/machine_util.go index fce97052c..e1b46db67 100644 --- a/pkg/util/provider/machinecontroller/machine_util.go +++ b/pkg/util/provider/machinecontroller/machine_util.go @@ -93,10 +93,9 @@ func UpdateMachineWithRetries(machineClient v1alpha1client.MachineInterface, mac */ // ValidateMachineClass validates the machine class. -func (c *controller) ValidateMachineClass(classSpec *v1alpha1.ClassSpec) (*v1alpha1.MachineClass, *v1.Secret, machineutils.RetryPeriod, error) { +func (c *controller) ValidateMachineClass(classSpec *v1alpha1.ClassSpec) (*v1alpha1.MachineClass, map[string][]byte, machineutils.RetryPeriod, error) { var ( machineClass *v1alpha1.MachineClass - secretRef *v1.Secret err error retry = machineutils.LongRetry ) @@ -118,16 +117,38 @@ func (c *controller) ValidateMachineClass(classSpec *v1alpha1.ClassSpec) (*v1alp return nil, nil, retry, err } - secretRef, err = c.getSecret(machineClass.SecretRef, machineClass.Name) + secretData, err := c.getSecretData(machineClass.Name, machineClass.SecretRef, machineClass.CredentialsSecretRef) if err != nil { - klog.Errorf("Secret not found for %q", machineClass.SecretRef.Name) + klog.V(2).Infof("Could not compute secret data: %+v", err) return nil, nil, retry, err } - return machineClass, secretRef, retry, nil + return machineClass, secretData, retry, nil } -// getSecret retrives the kubernetes secret if found +func (c *controller) getSecretData(machineClassName string, secretRefs ...*v1.SecretReference) (map[string][]byte, error) { + var secretData map[string][]byte + + for _, secretRef := range secretRefs { + if secretRef == nil { + continue + } + + secretRef, err := c.getSecret(secretRef, machineClassName) + if err != nil { + klog.V(2).Infof("Secret reference %s/%s not found", secretRef.Namespace, secretRef.Name) + return nil, err + } + + if secretRef != nil { + secretData = mergeDataMaps(secretData, secretRef.Data) + } + } + + return secretData, nil +} + +// getSecret retrieves the kubernetes secret if found func (c *controller) getSecret(ref *v1.SecretReference, MachineClassName string) (*v1.Secret, error) { if ref == nil { // If no secretRef, return nil @@ -161,6 +182,18 @@ func nodeConditionsHaveChanged(machineConditions []v1.NodeCondition, nodeConditi return false } +func mergeDataMaps(in map[string][]byte, maps ...map[string][]byte) map[string][]byte { + out := make(map[string][]byte) + + for _, m := range append([]map[string][]byte{in}, maps...) { + for k, v := range m { + out[k] = v + } + } + + return out +} + // syncMachineNodeTemplate syncs nodeTemplates between machine and corresponding node-object. // It ensures, that any nodeTemplate element available on Machine should be available on node-object. // Although there could be more elements already available on node-object which will not be touched. diff --git a/pkg/util/provider/machinecontroller/migrate_machineclass.go b/pkg/util/provider/machinecontroller/migrate_machineclass.go index faadb1559..0f5b79845 100644 --- a/pkg/util/provider/machinecontroller/migrate_machineclass.go +++ b/pkg/util/provider/machinecontroller/migrate_machineclass.go @@ -5,17 +5,17 @@ import ( "context" "fmt" - "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1" - "github.com/gardener/machine-controller-manager/pkg/util/provider/driver" - "github.com/gardener/machine-controller-manager/pkg/util/provider/machinecodes/codes" - "github.com/gardener/machine-controller-manager/pkg/util/provider/machinecodes/status" - "github.com/gardener/machine-controller-manager/pkg/util/provider/machineutils" - v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog" + + "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1" + "github.com/gardener/machine-controller-manager/pkg/util/provider/driver" + "github.com/gardener/machine-controller-manager/pkg/util/provider/machinecodes/codes" + "github.com/gardener/machine-controller-manager/pkg/util/provider/machinecodes/status" + "github.com/gardener/machine-controller-manager/pkg/util/provider/machineutils" ) const ( @@ -110,7 +110,7 @@ func (c *controller) createMachineClass(providerSpecificMachineClass interface{} } } else if err != nil { - // Anyother kind of error while fetching the machineClass object + // Another kind of error while fetching the machineClass object return machineutils.ShortRetry, err } @@ -387,7 +387,7 @@ func (c *controller) addMigratedAnnotationForProviderMachineClass(classSpec *v1a } // TryMachineClassMigration tries to migrate the provider-specific machine class to the generic machine-class. -func (c *controller) TryMachineClassMigration(classSpec *v1alpha1.ClassSpec) (*v1alpha1.MachineClass, *v1.Secret, machineutils.RetryPeriod, error) { +func (c *controller) TryMachineClassMigration(classSpec *v1alpha1.ClassSpec) (*v1alpha1.MachineClass, map[string][]byte, machineutils.RetryPeriod, error) { var ( err error providerSpecificMachineClass interface{} diff --git a/pkg/util/provider/machinecontroller/secret.go b/pkg/util/provider/machinecontroller/secret.go index a94267832..3d2831d8e 100644 --- a/pkg/util/provider/machinecontroller/secret.go +++ b/pkg/util/provider/machinecontroller/secret.go @@ -22,10 +22,12 @@ import ( "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1" corev1 "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" "k8s.io/klog" ) @@ -159,12 +161,32 @@ func (c *controller) enqueueSecretAfter(obj interface{}, after time.Duration) { c.secretQueue.AddAfter(key, after) } +func enqueueSecretForReferences(queue workqueue.RateLimitingInterface, secretRefs ...*corev1.SecretReference) { + for _, secretRef := range secretRefs { + if secretRef != nil { + queue.Add(secretRef.Namespace + "/" + secretRef.Name) + } + } +} + +func enqueueSecretForReferenceIfChanged(queue workqueue.RateLimitingInterface, oldSecretRef, newSecretRef *corev1.SecretReference) { + if !apiequality.Semantic.DeepEqual(oldSecretRef, newSecretRef) { + if oldSecretRef != nil { + queue.Add(oldSecretRef.Namespace + "/" + oldSecretRef.Name) + } + if newSecretRef != nil { + queue.Add(newSecretRef.Namespace + "/" + newSecretRef.Name) + } + } +} + func (c *controller) machineClassToSecretAdd(obj interface{}) { machineClass, ok := obj.(*v1alpha1.MachineClass) if !ok || machineClass == nil || machineClass.SecretRef == nil { return } - c.secretQueue.Add(machineClass.SecretRef.Namespace + "/" + machineClass.SecretRef.Name) + + enqueueSecretForReferences(c.secretQueue, machineClass.SecretRef, machineClass.CredentialsSecretRef) } func (c *controller) machineClassToSecretUpdate(oldObj interface{}, newObj interface{}) { @@ -177,11 +199,8 @@ func (c *controller) machineClassToSecretUpdate(oldObj interface{}, newObj inter return } - if oldMachineClass.SecretRef.Name != newMachineClass.SecretRef.Name || - oldMachineClass.SecretRef.Namespace != newMachineClass.SecretRef.Namespace { - c.secretQueue.Add(oldMachineClass.SecretRef.Namespace + "/" + oldMachineClass.SecretRef.Name) - c.secretQueue.Add(newMachineClass.SecretRef.Namespace + "/" + newMachineClass.SecretRef.Name) - } + enqueueSecretForReferenceIfChanged(c.secretQueue, oldMachineClass.SecretRef, newMachineClass.SecretRef) + enqueueSecretForReferenceIfChanged(c.secretQueue, oldMachineClass.CredentialsSecretRef, newMachineClass.CredentialsSecretRef) } func (c *controller) machineClassToSecretDelete(obj interface{}) { diff --git a/pkg/util/provider/machinecontroller/secret_util.go b/pkg/util/provider/machinecontroller/secret_util.go index 7e8a3f91e..77913541a 100644 --- a/pkg/util/provider/machinecontroller/secret_util.go +++ b/pkg/util/provider/machinecontroller/secret_util.go @@ -47,7 +47,8 @@ func (c *controller) findMachineClassForSecret(name string) ([]*v1alpha1.Machine } var filtered []*v1alpha1.MachineClass for _, machineClass := range machineClasses { - if machineClass.SecretRef != nil && machineClass.SecretRef.Name == name { + if (machineClass.SecretRef != nil && machineClass.SecretRef.Name == name) || + (machineClass.CredentialsSecretRef != nil && machineClass.CredentialsSecretRef.Name == name) { filtered = append(filtered, machineClass) } }