Add Ability to Specify Kubelet Configuration, specifically clusterDNS (…

…#1013)

* Add ability to specify ClusterDNSIP

Signed-off-by: Michael Irwin <[email protected]>

Resolves #928

* Add unit test to validate clusterDNSIP setting

Signed-off-by: Michael Irwin <[email protected]>

* Update pkg/apis/provisioning/v1alpha5/constraints.go

Co-authored-by: Ellis Tarn <[email protected]>

* Rename kubelet args to align with kubelet types

Signed-off-by: Michael Irwin <[email protected]>

* CRD docs update

Signed-off-by: Michael Irwin <[email protected]>

* Refactored userData creation to reduce code complexity

Signed-off-by: Michael Irwin <[email protected]>

* Rename kubeletArgs to kubeletConfiguration

Signed-off-by: Michael Irwin <[email protected]>

* Fix mismatch between struct name and JSON key

Signed-off-by: Michael Irwin <[email protected]>

Co-authored-by: Ellis Tarn <[email protected]>

charts/karpenter/crds/karpenter.sh_provisioners.yaml

@@ -40,6 +40,17 @@ spec:
               Node properties are determined from a combination of provisioner and
               pod scheduling constraints.
             properties:
+              kubeletConfiguration:
+                description: KubeletConfiguration are options passed to the kubelet
+                  when provisioning nodes
+                properties:
+                  clusterDNS:
+                    description: clusterDNS is a list of IP addresses for the cluster
+                      DNS server. Note that not all providers may use all addresses.
+                    items:
+                      type: string
+                    type: array
+                type: object
               labels:
                 additionalProperties:
                   type: string

pkg/apis/provisioning/v1alpha5/constraints.go

@@ -34,6 +34,9 @@ type Constraints struct {
 	Taints Taints `json:"taints,omitempty"`
 	// Requirements are layered with Labels and applied to every node.
 	Requirements Requirements `json:"requirements,omitempty"`
+	// KubeletConfiguration are options passed to the kubelet when provisioning nodes
+	//+optional
+	KubeletConfiguration KubeletConfiguration `json:"kubeletConfiguration,omitempty"`
 	// Provider contains fields specific to your cloudprovider.
 	// +kubebuilder:pruning:PreserveUnknownFields
 	Provider *runtime.RawExtension `json:"provider,omitempty"`
@@ -64,9 +67,10 @@ func (c *Constraints) ValidatePod(pod *v1.Pod) error {
 
 func (c *Constraints) Tighten(pod *v1.Pod) *Constraints {
 	return &Constraints{
-		Labels:       c.Labels,
-		Requirements: c.Requirements.With(PodRequirements(pod)).Consolidate().WellKnown(),
-		Taints:       c.Taints,
-		Provider:     c.Provider,
+		Labels:               c.Labels,
+		Requirements:         c.Requirements.With(PodRequirements(pod)).Consolidate().WellKnown(),
+		Taints:               c.Taints,
+		Provider:             c.Provider,
+		KubeletConfiguration: c.KubeletConfiguration,
 	}
 }

pkg/apis/provisioning/v1alpha5/kubelet_configuration.go

@@ -0,0 +1,25 @@
+/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha5
+
+// KubeletConfiguration defines args to be used when configuring kubelet on provisioned nodes.
+// They are a subset of the upstream types, recognizing not all options may be supported.
+// Wherever possible, the types and names should reflect the upstream kubelet types.
+type KubeletConfiguration struct {
+	// clusterDNS is a list of IP addresses for the cluster DNS server.
+	// Note that not all providers may use all addresses.
+	//+optional
+	ClusterDNS []string `json:"clusterDNS,omitempty"`
+}

pkg/cloudprovider/aws/launchtemplate.go

@@ -247,7 +247,22 @@ exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1
 			*caBundle))
 	}
 
-	nodeLabels := functional.UnionStringMaps(additionalLabels, constraints.Labels)
+	nodeLabelArgs := p.getNodeLabelArgs(functional.UnionStringMaps(additionalLabels, constraints.Labels))
+	nodeTaintsArgs := p.getNodeTaintArgs(constraints)
+	kubeletExtraArgs := strings.Trim(strings.Join([]string{nodeLabelArgs, nodeTaintsArgs.String()}, " "), " ")
+
+	if len(kubeletExtraArgs) > 0 {
+		userData.WriteString(fmt.Sprintf(` \
+    --kubelet-extra-args '%s'`, kubeletExtraArgs))
+	}
+	if len(constraints.KubeletConfiguration.ClusterDNS) > 0 {
+		userData.WriteString(fmt.Sprintf(` \
+    --dns-cluster-ip '%s'`, constraints.KubeletConfiguration.ClusterDNS[0]))
+	}
+	return base64.StdEncoding.EncodeToString(userData.Bytes()), nil
+}
+
+func (p *LaunchTemplateProvider) getNodeLabelArgs(nodeLabels map[string]string) string {
 	nodeLabelArgs := ""
 	if len(nodeLabels) > 0 {
 		labelStrings := []string{}
@@ -261,6 +276,10 @@ exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1
 		}
 		nodeLabelArgs = fmt.Sprintf("--node-labels=%s", strings.Join(labelStrings, ","))
 	}
+	return nodeLabelArgs
+}
+
+func (p *LaunchTemplateProvider) getNodeTaintArgs(constraints *v1alpha1.Constraints) bytes.Buffer {
 	var nodeTaintsArgs bytes.Buffer
 	if len(constraints.Taints) > 0 {
 		nodeTaintsArgs.WriteString("--register-with-taints=")
@@ -276,12 +295,7 @@ exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1
 			nodeTaintsArgs.WriteString(fmt.Sprintf("%s=%s:%s", taint.Key, taint.Value, taint.Effect))
 		}
 	}
-	kubeletExtraArgs := strings.Trim(strings.Join([]string{nodeLabelArgs, nodeTaintsArgs.String()}, " "), " ")
-	if len(kubeletExtraArgs) > 0 {
-		userData.WriteString(fmt.Sprintf(` \
-    --kubelet-extra-args '%s'`, kubeletExtraArgs))
-	}
-	return base64.StdEncoding.EncodeToString(userData.Bytes()), nil
+	return nodeTaintsArgs
 }
 
 func (p *LaunchTemplateProvider) GetCABundle(ctx context.Context) (*string, error) {

pkg/cloudprovider/aws/suite_test.go

@@ -16,6 +16,7 @@ package aws
 
 import (
 	"context"
+	"encoding/base64"
 	"encoding/json"
 	"testing"
 
@@ -407,6 +408,17 @@ var _ = Describe("Allocation", func() {
 				))
 			})
 		})
+		Context("Kubelet Args", func() {
+			It("should specify the --dns-cluster-ip flag when clusterDNSIP is set", func() {
+				provisioner.Spec.KubeletConfiguration.ClusterDNS = []string{"10.0.10.100"}
+				pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, ProvisionerWithProvider(provisioner, provider), test.UnschedulablePod())[0]
+				ExpectScheduled(ctx, env.Client, pod)
+				Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Cardinality()).To(Equal(1))
+				input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop().(*ec2.CreateLaunchTemplateInput)
+				userData, _ := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData)
+				Expect(string(userData)).To(ContainSubstring("--dns-cluster-ip '10.0.10.100'"))
+			})
+		})
 	})
 	Context("Defaulting", func() {
 		It("should default subnetSelector", func() {

website/content/en/docs/provisioner.md

@@ -137,6 +137,18 @@ Karpenter supports `amd64` nodes, and `arm64` nodes.
 Karpenter supports specifying capacity type, which is analogous to [EC2 purchase options](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-purchasing-options.html).
 
 
+## spec.kubeletConfiguration
+
+Karpenter provides the ability to specify a few additional Kubelet args. These are all optional and provide support for 
+additional customization and use cases. Adjust these only if you know you need to do so.
+
+```yaml
+spec:
+  kubeletConfiguration:
+    clusterDNS: ["10.0.1.100"]
+```
+
+
 ## spec.provider
 
 This section is cloud provider specific. Reference the appropriate documentation: