Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add Ability to Specify Kubelet Configuration, specifically clusterDNS #1013

Merged
merged 8 commits into from
Dec 17, 2021
Merged
11 changes: 11 additions & 0 deletions charts/karpenter/crds/karpenter.sh_provisioners.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,17 @@ spec:
Node properties are determined from a combination of provisioner and
pod scheduling constraints.
properties:
kubeletConfiguration:
description: KubeletConfiguration are options passed to the kubelet
when provisioning nodes
properties:
clusterDns:
description: clusterDNS is a list of IP addresses for the cluster
DNS server. Note that not all providers may use all addresses.
items:
type: string
type: array
type: object
labels:
additionalProperties:
type: string
Expand Down
12 changes: 8 additions & 4 deletions pkg/apis/provisioning/v1alpha5/constraints.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@ type Constraints struct {
Taints Taints `json:"taints,omitempty"`
// Requirements are layered with Labels and applied to every node.
Requirements Requirements `json:"requirements,omitempty"`
// KubeletConfiguration are options passed to the kubelet when provisioning nodes
//+optional
KubeletConfiguration KubeletConfiguration `json:"kubeletConfiguration,omitempty"`
// Provider contains fields specific to your cloudprovider.
// +kubebuilder:pruning:PreserveUnknownFields
Provider *runtime.RawExtension `json:"provider,omitempty"`
Expand Down Expand Up @@ -64,9 +67,10 @@ func (c *Constraints) ValidatePod(pod *v1.Pod) error {

func (c *Constraints) Tighten(pod *v1.Pod) *Constraints {
return &Constraints{
Labels: c.Labels,
Requirements: c.Requirements.With(PodRequirements(pod)).Consolidate().WellKnown(),
Taints: c.Taints,
Provider: c.Provider,
Labels: c.Labels,
Requirements: c.Requirements.With(PodRequirements(pod)).Consolidate().WellKnown(),
Taints: c.Taints,
Provider: c.Provider,
KubeletConfiguration: c.KubeletConfiguration,
}
}
25 changes: 25 additions & 0 deletions pkg/apis/provisioning/v1alpha5/kubelet_configuration.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha5

// KubeletConfiguration defines args to be used when configuring kubelet on provisioned nodes.
// They are a subset of the upstream types, recognizing not all options may be supported.
// Wherever possible, the types and names should reflect the upstream kubelet types.
type KubeletConfiguration struct {
// clusterDNS is a list of IP addresses for the cluster DNS server.
// Note that not all providers may use all addresses.
//+optional
ClusterDNS []string `json:"clusterDns,omitempty"`
}
21 changes: 21 additions & 0 deletions pkg/apis/provisioning/v1alpha5/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

28 changes: 21 additions & 7 deletions pkg/cloudprovider/aws/launchtemplate.go
Original file line number Diff line number Diff line change
Expand Up @@ -247,7 +247,22 @@ exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1
*caBundle))
}

nodeLabels := functional.UnionStringMaps(additionalLabels, constraints.Labels)
nodeLabelArgs := p.getNodeLabelArgs(functional.UnionStringMaps(additionalLabels, constraints.Labels))
nodeTaintsArgs := p.getNodeTaintArgs(constraints)
kubeletExtraArgs := strings.Trim(strings.Join([]string{nodeLabelArgs, nodeTaintsArgs.String()}, " "), " ")

if len(kubeletExtraArgs) > 0 {
userData.WriteString(fmt.Sprintf(` \
--kubelet-extra-args '%s'`, kubeletExtraArgs))
}
if len(constraints.KubeletConfiguration.ClusterDNS) > 0 {
userData.WriteString(fmt.Sprintf(` \
--dns-cluster-ip '%s'`, constraints.KubeletConfiguration.ClusterDNS[0]))
}
return base64.StdEncoding.EncodeToString(userData.Bytes()), nil
}

func (p *LaunchTemplateProvider) getNodeLabelArgs(nodeLabels map[string]string) string {
nodeLabelArgs := ""
if len(nodeLabels) > 0 {
labelStrings := []string{}
Expand All @@ -261,6 +276,10 @@ exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1
}
nodeLabelArgs = fmt.Sprintf("--node-labels=%s", strings.Join(labelStrings, ","))
}
return nodeLabelArgs
}

func (p *LaunchTemplateProvider) getNodeTaintArgs(constraints *v1alpha1.Constraints) bytes.Buffer {
var nodeTaintsArgs bytes.Buffer
if len(constraints.Taints) > 0 {
nodeTaintsArgs.WriteString("--register-with-taints=")
Expand All @@ -276,12 +295,7 @@ exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1
nodeTaintsArgs.WriteString(fmt.Sprintf("%s=%s:%s", taint.Key, taint.Value, taint.Effect))
}
}
kubeletExtraArgs := strings.Trim(strings.Join([]string{nodeLabelArgs, nodeTaintsArgs.String()}, " "), " ")
if len(kubeletExtraArgs) > 0 {
userData.WriteString(fmt.Sprintf(` \
--kubelet-extra-args '%s'`, kubeletExtraArgs))
}
return base64.StdEncoding.EncodeToString(userData.Bytes()), nil
return nodeTaintsArgs
}

func (p *LaunchTemplateProvider) GetCABundle(ctx context.Context) (*string, error) {
Expand Down
12 changes: 12 additions & 0 deletions pkg/cloudprovider/aws/suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ package aws

import (
"context"
"encoding/base64"
"encoding/json"
"testing"

Expand Down Expand Up @@ -407,6 +408,17 @@ var _ = Describe("Allocation", func() {
))
})
})
Context("Kubelet Args", func() {
It("should specify the --dns-cluster-ip flag when clusterDNSIP is set", func() {
provisioner.Spec.KubeletConfiguration.ClusterDNS = []string{"10.0.10.100"}
pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, ProvisionerWithProvider(provisioner, provider), test.UnschedulablePod())[0]
ExpectScheduled(ctx, env.Client, pod)
Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Cardinality()).To(Equal(1))
input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop().(*ec2.CreateLaunchTemplateInput)
userData, _ := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData)
Expect(string(userData)).To(ContainSubstring("--dns-cluster-ip '10.0.10.100'"))
})
})
})
Context("Defaulting", func() {
It("should default subnetSelector", func() {
Expand Down
12 changes: 12 additions & 0 deletions website/content/en/docs/provisioner.md
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,18 @@ Karpenter supports `amd64` nodes, and `arm64` nodes.
Karpenter supports specifying capacity type, which is analogous to [EC2 purchase options](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-purchasing-options.html).


## spec.kubeletConfiguration

Karpenter provides the ability to specify a few additional Kubelet args. These are all optional and provide support for
additional customization and use cases. Adjust these only if you know you need to do so.

```yaml
spec:
kubeletConfiguration:
clusterDNS: ["10.0.1.100"]
```


## spec.provider

This section is cloud provider specific. Reference the appropriate documentation:
Expand Down