Merge pull request #10821 from seh/automated-cherry-pick-of-#10733-or…

…igin-release-1.20

Automated cherry pick of #10733: Define "ManageStorageClasses" cloud config field

k8s/crds/kops.k8s.io_clusters.yaml

@@ -277,6 +277,11 @@ spec:
                     description: GCEServiceAccount specifies the service account with
                       which the GCE VM runs
                     type: string
+                  manageStorageClasses:
+                    description: ManageStorageClasses specifies whether kOps should
+                      create and maintain a set of StorageClasses, one of which it
+                      nominates as the default class for the cluster.
+                    type: boolean
                   multizone:
                     description: GCE cloud-config options
                     type: boolean

nodeup/pkg/model/tests/golden/awsiam/tasks-kube-apiserver.yaml

@@ -42,6 +42,7 @@ contents: |
       - --authorization-mode=AlwaysAllow
       - --bind-address=0.0.0.0
       - --client-ca-file=/srv/kubernetes/ca.crt
+      - --cloud-config=/etc/kubernetes/cloud.config
       - --cloud-provider=aws
       - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,NodeRestriction,ResourceQuota
       - --etcd-cafile=/etc/kubernetes/pki/kube-apiserver/etcd-ca.crt
@@ -121,6 +122,9 @@ contents: |
         readOnly: true
       - mountPath: /etc/kubernetes/pki/kube-apiserver
         name: pki
+      - mountPath: /etc/kubernetes/cloud.config
+        name: cloudconfig
+        readOnly: true
       - mountPath: /srv/kubernetes
         name: srvkube
         readOnly: true
@@ -170,6 +174,9 @@ contents: |
         path: /etc/kubernetes/pki/kube-apiserver
         type: DirectoryOrCreate
       name: pki
+    - hostPath:
+        path: /etc/kubernetes/cloud.config
+      name: cloudconfig
     - hostPath:
         path: /srv/kubernetes
       name: srvkube

nodeup/pkg/model/tests/golden/minimal/tasks-kube-apiserver.yaml

@@ -20,6 +20,7 @@ contents: |
       - --authorization-mode=AlwaysAllow
       - --bind-address=0.0.0.0
       - --client-ca-file=/srv/kubernetes/ca.crt
+      - --cloud-config=/etc/kubernetes/cloud.config
       - --cloud-provider=aws
       - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,NodeRestriction,ResourceQuota
       - --etcd-cafile=/etc/kubernetes/pki/kube-apiserver/etcd-ca.crt
@@ -99,6 +100,9 @@ contents: |
         readOnly: true
       - mountPath: /etc/kubernetes/pki/kube-apiserver
         name: pki
+      - mountPath: /etc/kubernetes/cloud.config
+        name: cloudconfig
+        readOnly: true
       - mountPath: /srv/kubernetes
         name: srvkube
         readOnly: true
@@ -145,6 +149,9 @@ contents: |
         path: /etc/kubernetes/pki/kube-apiserver
         type: DirectoryOrCreate
       name: pki
+    - hostPath:
+        path: /etc/kubernetes/cloud.config
+      name: cloudconfig
     - hostPath:
         path: /srv/kubernetes
       name: srvkube

nodeup/pkg/model/tests/golden/minimal/tasks-kube-controller-manager.yaml

@@ -14,6 +14,7 @@ contents: |
     - args:
       - --allocate-node-cidrs=true
       - --attach-detach-reconcile-sync-period=1m0s
+      - --cloud-config=/etc/kubernetes/cloud.config
       - --cloud-provider=aws
       - --cluster-cidr=100.96.0.0/11
       - --cluster-name=minimal.example.com
@@ -74,6 +75,9 @@ contents: |
       - mountPath: /etc/openssl
         name: etcopenssl
         readOnly: true
+      - mountPath: /etc/kubernetes/cloud.config
+        name: cloudconfig
+        readOnly: true
       - mountPath: /srv/kubernetes
         name: srvkube
         readOnly: true
@@ -118,6 +122,9 @@ contents: |
     - hostPath:
         path: /etc/openssl
       name: etcopenssl
+    - hostPath:
+        path: /etc/kubernetes/cloud.config
+      name: cloudconfig
     - hostPath:
         path: /srv/kubernetes
       name: srvkube

nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-amd64.yaml

@@ -20,6 +20,7 @@ contents: |
       - --authorization-mode=AlwaysAllow
       - --bind-address=0.0.0.0
       - --client-ca-file=/srv/kubernetes/ca.crt
+      - --cloud-config=/etc/kubernetes/cloud.config
       - --cloud-provider=aws
       - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,NodeRestriction,ResourceQuota
       - --etcd-cafile=/etc/kubernetes/pki/kube-apiserver/etcd-ca.crt
@@ -99,6 +100,9 @@ contents: |
         readOnly: true
       - mountPath: /etc/kubernetes/pki/kube-apiserver
         name: pki
+      - mountPath: /etc/kubernetes/cloud.config
+        name: cloudconfig
+        readOnly: true
       - mountPath: /srv/kubernetes
         name: srvkube
         readOnly: true
@@ -145,6 +149,9 @@ contents: |
         path: /etc/kubernetes/pki/kube-apiserver
         type: DirectoryOrCreate
       name: pki
+    - hostPath:
+        path: /etc/kubernetes/cloud.config
+      name: cloudconfig
     - hostPath:
         path: /srv/kubernetes
       name: srvkube

nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-arm64.yaml

@@ -20,6 +20,7 @@ contents: |
       - --authorization-mode=AlwaysAllow
       - --bind-address=0.0.0.0
       - --client-ca-file=/srv/kubernetes/ca.crt
+      - --cloud-config=/etc/kubernetes/cloud.config
       - --cloud-provider=aws
       - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,NodeRestriction,ResourceQuota
       - --etcd-cafile=/etc/kubernetes/pki/kube-apiserver/etcd-ca.crt
@@ -99,6 +100,9 @@ contents: |
         readOnly: true
       - mountPath: /etc/kubernetes/pki/kube-apiserver
         name: pki
+      - mountPath: /etc/kubernetes/cloud.config
+        name: cloudconfig
+        readOnly: true
       - mountPath: /srv/kubernetes
         name: srvkube
         readOnly: true
@@ -145,6 +149,9 @@ contents: |
         path: /etc/kubernetes/pki/kube-apiserver
         type: DirectoryOrCreate
       name: pki
+    - hostPath:
+        path: /etc/kubernetes/cloud.config
+      name: cloudconfig
     - hostPath:
         path: /srv/kubernetes
       name: srvkube

nodeup/pkg/model/tests/golden/side-loading/tasks-kube-controller-manager-amd64.yaml

@@ -14,6 +14,7 @@ contents: |
     - args:
       - --allocate-node-cidrs=true
       - --attach-detach-reconcile-sync-period=1m0s
+      - --cloud-config=/etc/kubernetes/cloud.config
       - --cloud-provider=aws
       - --cluster-cidr=100.96.0.0/11
       - --cluster-name=minimal.example.com
@@ -74,6 +75,9 @@ contents: |
       - mountPath: /etc/openssl
         name: etcopenssl
         readOnly: true
+      - mountPath: /etc/kubernetes/cloud.config
+        name: cloudconfig
+        readOnly: true
       - mountPath: /srv/kubernetes
         name: srvkube
         readOnly: true
@@ -118,6 +122,9 @@ contents: |
     - hostPath:
         path: /etc/openssl
       name: etcopenssl
+    - hostPath:
+        path: /etc/kubernetes/cloud.config
+      name: cloudconfig
     - hostPath:
         path: /srv/kubernetes
       name: srvkube

nodeup/pkg/model/tests/golden/side-loading/tasks-kube-controller-manager-arm64.yaml

@@ -14,6 +14,7 @@ contents: |
     - args:
       - --allocate-node-cidrs=true
       - --attach-detach-reconcile-sync-period=1m0s
+      - --cloud-config=/etc/kubernetes/cloud.config
       - --cloud-provider=aws
       - --cluster-cidr=100.96.0.0/11
       - --cluster-name=minimal.example.com
@@ -74,6 +75,9 @@ contents: |
       - mountPath: /etc/openssl
         name: etcopenssl
         readOnly: true
+      - mountPath: /etc/kubernetes/cloud.config
+        name: cloudconfig
+        readOnly: true
       - mountPath: /srv/kubernetes
         name: srvkube
         readOnly: true
@@ -118,6 +122,9 @@ contents: |
     - hostPath:
         path: /etc/openssl
       name: etcopenssl
+    - hostPath:
+        path: /etc/kubernetes/cloud.config
+      name: cloudconfig
     - hostPath:
         path: /srv/kubernetes
       name: srvkube

pkg/apis/kops/componentconfig.go

@@ -792,6 +792,12 @@ type AzureConfiguration struct {
 
 // CloudConfiguration defines the cloud provider configuration
 type CloudConfiguration struct {
+	// Cross-cloud provider options
+
+	// ManageStorageClasses specifies whether kOps should create and maintain a set of
+	// StorageClasses, one of which it nominates as the default class for the cluster.
+	ManageStorageClasses *bool `json:"manageStorageClasses,omitempty"`
+
 	// GCE cloud-config options
 	Multizone          *bool   `json:"multizone,omitempty"`
 	NodeTags           *string `json:"nodeTags,omitempty"`

pkg/apis/kops/v1alpha2/componentconfig.go

@@ -791,6 +791,12 @@ type AzureConfiguration struct {
 
 // CloudConfiguration defines the cloud provider configuration
 type CloudConfiguration struct {
+	// Cross-cloud provider options
+
+	// ManageStorageClasses specifies whether kOps should create and maintain a set of
+	// StorageClasses, one of which it nominates as the default class for the cluster.
+	ManageStorageClasses *bool `json:"manageStorageClasses,omitempty"`
+
 	// GCE cloud-config options
 	Multizone          *bool   `json:"multizone,omitempty"`
 	NodeTags           *string `json:"nodeTags,omitempty"`

pkg/apis/kops/validation/validation.go

@@ -242,6 +242,10 @@ func validateClusterSpec(spec *kops.ClusterSpec, c *kops.Cluster, fieldPath *fie
 		}
 	}
 
+	if spec.CloudConfig != nil {
+		allErrs = append(allErrs, validateCloudConfiguration(spec.CloudConfig, fieldPath.Child("cloudConfig"))...)
+	}
+
 	return allErrs
 }
 
@@ -1302,3 +1306,14 @@ func validateAWSLoadBalancerController(cluster *kops.Cluster, spec *kops.AWSLoad
 	}
 	return allErrs
 }
+
+func validateCloudConfiguration(cloudConfig *kops.CloudConfiguration, fldPath *field.Path) (allErrs field.ErrorList) {
+	if cloudConfig.ManageStorageClasses != nil && cloudConfig.Openstack != nil &&
+		cloudConfig.Openstack.BlockStorage != nil && cloudConfig.Openstack.BlockStorage.CreateStorageClass != nil {
+		if *cloudConfig.Openstack.BlockStorage.CreateStorageClass != *cloudConfig.ManageStorageClasses {
+			allErrs = append(allErrs, field.Forbidden(fldPath.Child("manageStorageClasses"),
+				"Management of storage classes and OpenStack block storage classes are both specified but disagree"))
+		}
+	}
+	return allErrs
+}

pkg/apis/kops/validation/validation_test.go

@@ -1033,3 +1033,96 @@ func Test_Validate_NodeLocalDNS(t *testing.T) {
 		testErrors(t, g.Input, errs, g.ExpectedErrors)
 	}
 }
+
+func Test_Validate_CloudConfiguration(t *testing.T) {
+	grid := []struct {
+		Description    string
+		Input          kops.CloudConfiguration
+		ExpectedErrors []string
+	}{
+		{
+			Description: "neither",
+			Input:       kops.CloudConfiguration{},
+		},
+		{
+			Description: "all false",
+			Input: kops.CloudConfiguration{
+				ManageStorageClasses: fi.Bool(false),
+			},
+		},
+		{
+			Description: "all true",
+			Input: kops.CloudConfiguration{
+				ManageStorageClasses: fi.Bool(true),
+			},
+		},
+		{
+			Description: "os false",
+			Input: kops.CloudConfiguration{
+				Openstack: &kops.OpenstackConfiguration{
+					BlockStorage: &kops.OpenstackBlockStorageConfig{
+						CreateStorageClass: fi.Bool(false),
+					},
+				}},
+		},
+		{
+			Description: "os false",
+			Input: kops.CloudConfiguration{
+				Openstack: &kops.OpenstackConfiguration{
+					BlockStorage: &kops.OpenstackBlockStorageConfig{
+						CreateStorageClass: fi.Bool(true),
+					},
+				}},
+		},
+		{
+			Description: "all false, os false",
+			Input: kops.CloudConfiguration{
+				ManageStorageClasses: fi.Bool(false),
+				Openstack: &kops.OpenstackConfiguration{
+					BlockStorage: &kops.OpenstackBlockStorageConfig{
+						CreateStorageClass: fi.Bool(false),
+					},
+				}},
+		},
+		{
+			Description: "all false, os true",
+			Input: kops.CloudConfiguration{
+				ManageStorageClasses: fi.Bool(false),
+				Openstack: &kops.OpenstackConfiguration{
+					BlockStorage: &kops.OpenstackBlockStorageConfig{
+						CreateStorageClass: fi.Bool(true),
+					},
+				}},
+			ExpectedErrors: []string{"Forbidden::cloudConfig.manageStorageClasses"},
+		},
+		{
+			Description: "all true, os false",
+			Input: kops.CloudConfiguration{
+				ManageStorageClasses: fi.Bool(true),
+				Openstack: &kops.OpenstackConfiguration{
+					BlockStorage: &kops.OpenstackBlockStorageConfig{
+						CreateStorageClass: fi.Bool(false),
+					},
+				}},
+			ExpectedErrors: []string{"Forbidden::cloudConfig.manageStorageClasses"},
+		},
+		{
+			Description: "all true, os true",
+			Input: kops.CloudConfiguration{
+				ManageStorageClasses: fi.Bool(true),
+				Openstack: &kops.OpenstackConfiguration{
+					BlockStorage: &kops.OpenstackBlockStorageConfig{
+						CreateStorageClass: fi.Bool(true),
+					},
+				}},
+		},
+	}
+
+	for _, g := range grid {
+		fldPath := field.NewPath("cloudConfig")
+		t.Run(g.Description, func(t *testing.T) {
+			errs := validateCloudConfiguration(&g.Input, fldPath)
+			testErrors(t, g.Input, errs, g.ExpectedErrors)
+		})
+	}
+}