Allow kOps to set labels required for cluster operations (#999)

* Allow kOps to set labels required for cluster operations

* Filter allowed flags from kubelet args

pkg/apis/provisioning/v1alpha5/provisioner_validation.go

@@ -90,6 +90,9 @@ func (c *Constraints) validateLabels() (errs *apis.FieldError) {
 
 func IsRestrictedLabelDomain(key string) bool {
 	labelDomain := getLabelDomain(key)
+	if AllowedLabelDomains.Has(labelDomain) {
+		return false
+	}
 	for restrictedLabelDomain := range RestrictedLabelDomains {
 		if strings.HasSuffix(labelDomain, restrictedLabelDomain) {
 			return true

pkg/apis/provisioning/v1alpha5/register.go

@@ -46,7 +46,16 @@ var (
 		EmptinessTimestampAnnotationKey,
 		v1.LabelHostname,
 	)
+
+	// AllowedLabelDomains are domains that may be restricted, but that is allowed because
+	// they are not used in a context where they may be passed as argument to kubelet.
+	// AllowedLabelDomains are evaluated before RestrictedLabelDomains
+	AllowedLabelDomains = sets.NewString(
+		"kops.k8s.io",
+	)
+
 	// These are either prohibited by the kubelet or reserved by karpenter
+	// They are evaluated after AllowedLabelDomains
 	KarpenterLabelDomain   = "karpenter.sh"
 	RestrictedLabelDomains = sets.NewString(
 		"kubernetes.io",

pkg/apis/provisioning/v1alpha5/suite_test.go

@@ -103,6 +103,13 @@ var _ = Describe("Validation", func() {
 				Expect(provisioner.Validate(ctx)).ToNot(Succeed())
 			}
 		})
+		It("should allow labels kOps require", func() {
+			provisioner.Spec.Labels = map[string]string{
+				"kops.k8s.io/instancegroup": "karpenter-nodes",
+				"kops.k8s.io/gpu":           "1",
+			}
+			Expect(provisioner.Validate(ctx)).To(Succeed())
+		})
 	})
 	Context("Taints", func() {
 		It("should succeed for valid taints", func() {

pkg/cloudprovider/aws/launchtemplate.go

@@ -26,6 +26,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
+	"github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5"
 	"github.com/aws/karpenter/pkg/cloudprovider"
 	"github.com/aws/karpenter/pkg/cloudprovider/aws/apis/v1alpha1"
 	"github.com/aws/karpenter/pkg/utils/functional"
@@ -247,19 +248,18 @@ exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1
 	}
 
 	nodeLabels := functional.UnionStringMaps(additionalLabels, constraints.Labels)
-	var nodeLabelArgs bytes.Buffer
+	nodeLabelArgs := ""
 	if len(nodeLabels) > 0 {
-		nodeLabelArgs.WriteString("--node-labels=")
-		first := true
+		labelStrings := []string{}
 		// Must be in sorted order or else equivalent options won't
 		// hash the same
 		for _, k := range sortedKeys(nodeLabels) {
-			if !first {
-				nodeLabelArgs.WriteString(",")
+			if v1alpha5.AllowedLabelDomains.Has(k) {
+				continue
 			}
-			first = false
-			nodeLabelArgs.WriteString(fmt.Sprintf("%s=%v", k, nodeLabels[k]))
+			labelStrings = append(labelStrings, fmt.Sprintf("%s=%v", k, nodeLabels[k]))
 		}
+		nodeLabelArgs = fmt.Sprintf("--node-labels=%s", strings.Join(labelStrings, ","))
 	}
 	var nodeTaintsArgs bytes.Buffer
 	if len(constraints.Taints) > 0 {
@@ -276,7 +276,7 @@ exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1
 			nodeTaintsArgs.WriteString(fmt.Sprintf("%s=%s:%s", taint.Key, taint.Value, taint.Effect))
 		}
 	}
-	kubeletExtraArgs := strings.Trim(strings.Join([]string{nodeLabelArgs.String(), nodeTaintsArgs.String()}, " "), " ")
+	kubeletExtraArgs := strings.Trim(strings.Join([]string{nodeLabelArgs, nodeTaintsArgs.String()}, " "), " ")
 	if len(kubeletExtraArgs) > 0 {
 		userData.WriteString(fmt.Sprintf(` \
     --kubelet-extra-args '%s'`, kubeletExtraArgs))