Revert "Use controllerruntime to discover CABundle (#636)"

This reverts commit 7012c92.

pkg/apis/provisioning/v1alpha3/provisioner_defaults.go

@@ -17,23 +17,65 @@ package v1alpha3
 import (
 	"context"
 	"encoding/base64"
+	"errors"
+	"io/ioutil"
+	"os"
+)
 
-	controllerruntime "sigs.k8s.io/controller-runtime"
+const (
+	InClusterCABundlePath = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
 )
 
 // SetDefaults for the provisioner
 func (p *Provisioner) SetDefaults(ctx context.Context) {}
 
-func (c *Cluster) GetCABundle(ctx context.Context) (*string, error) {
+// SetDefaults for the provisioner, cascading to all subspecs
+func (s *ProvisionerSpec) SetDefaults(ctx context.Context) {}
+
+// WithDefaults returns a copy of this Provisioner with some empty/missing
+// properties replaced by (potentially dynamic) cloud provider agnostic default values.
+// The returned copy might be complemented by dynamic default values which
+// must not be hoisted (saved) into the original Provisioner CRD as those
+// default values might change over time (e.g. rolling upgrade of CABundle, ...).
+func (p *Provisioner) WithDynamicDefaults(ctx context.Context) (_ Provisioner, err error) {
+	provisioner := *p.DeepCopy()
+	provisioner.Spec, err = provisioner.Spec.withDynamicDefaults()
+	return provisioner, err
+}
+
+// WithDefaults returns a copy of this ProvisionerSpec with some empty
+// properties replaced by default values.
+func (s *ProvisionerSpec) withDynamicDefaults() (_ ProvisionerSpec, err error) {
+	spec := *s.DeepCopy()
+	spec.Cluster, err = spec.Cluster.withDynamicDefaults()
+	return spec, err
+}
+
+// WithDefaults returns a copy of this Cluster with some empty
+// properties replaced by default values. Notably, it will try
+// to load the CABundle from the in-cluster configuraiton if it
+// is not explicitly set.
+func (c *Cluster) withDynamicDefaults() (_ Cluster, err error) {
+	cluster := *c.DeepCopy()
+	cluster.CABundle, err = cluster.getCABundle()
+	return cluster, err
+}
+
+func (c *Cluster) getCABundle() (*string, error) {
 	if c.CABundle != nil {
-		// If CABundle is explicitly provided, use that one. An empty
-		// string is a valid value here if the intention is to disable
-		// the in-cluster CABundle, and using the HTTP client's
-		// default trust-store (CABundle) instead.
+		// If CABundle is explicitly provided use that one. An empty string is
+		// a valid value here if the intention is to disable the in-cluster CABundle
+		// and using the HTTP client's default trust-store (CABundle) instead.
 		return c.CABundle, nil
 	}
-
-	config := controllerruntime.GetConfigOrDie()
-	encoded := base64.StdEncoding.EncodeToString(config.CAData)
+	// Otherwise, fallback to the in-cluster configuration.
+	binary, err := ioutil.ReadFile(InClusterCABundlePath)
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return nil, nil
+		}
+		return nil, err
+	}
+	encoded := base64.StdEncoding.EncodeToString(binary)
 	return &encoded, nil
 }

pkg/cloudprovider/aws/cloudprovider.go

@@ -33,7 +33,6 @@ import (
 	v1 "k8s.io/api/core/v1"
 	"knative.dev/pkg/apis"
 	"knative.dev/pkg/logging"
-	"knative.dev/pkg/ptr"
 )
 
 const (
@@ -175,7 +174,7 @@ func (c *CloudProvider) ValidateConstraints(ctx context.Context, constraints *v1
 
 // Validate cloud provider specific components of the cluster spec.
 func (c *CloudProvider) ValidateSpec(ctx context.Context, spec *v1alpha3.ProvisionerSpec) (errs *apis.FieldError) {
-	if ptr.StringValue(spec.Cluster.Name) == "" {
+	if spec.Cluster.Name == nil || len(*spec.Cluster.Name) == 0 {
 		errs = errs.Also(apis.ErrMissingField("name")).ViaField("cluster")
 	}
 	return errs

pkg/cloudprovider/aws/launchtemplate.go

@@ -19,6 +19,7 @@ import (
 	"context"
 	"encoding/base64"
 	"fmt"
+	"text/template"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
@@ -34,6 +35,18 @@ import (
 
 const (
 	launchTemplateNameFormat = "Karpenter-%s-%s"
+	bottlerocketUserData     = `
+[settings.kubernetes]
+api-server = "{{.Cluster.Endpoint}}"
+{{if .Cluster.CABundle}}{{if len .Cluster.CABundle}}cluster-certificate = "{{.Cluster.CABundle}}"{{end}}{{end}}
+cluster-name = "{{if .Cluster.Name}}{{.Cluster.Name}}{{end}}"
+{{if .Constraints.Labels }}[settings.kubernetes.node-labels]{{ end }}
+{{ range $Key, $Value := .Constraints.Labels }}"{{ $Key }}" = "{{ $Value }}"
+{{ end }}
+{{if .Constraints.Taints }}[settings.kubernetes.node-taints]{{ end }}
+{{ range $Taint := .Constraints.Taints }}"{{ $Taint.Key }}" = "{{ $Taint.Value}}:{{ $Taint.Effect }}"
+{{ end }}
+`
 )
 
 type LaunchTemplateProvider struct {
@@ -90,16 +103,10 @@ func (p *LaunchTemplateProvider) Get(ctx context.Context, provisioner *v1alpha3.
 		return nil, err
 	}
 
-	// 3. Get userData for Node
-	userData, err := p.getUserData(ctx, provisioner, constraints)
-	if err != nil {
-		return nil, err
-	}
-
 	// 4. Ensure the launch template exists, or create it
 	launchTemplate, err := p.ensureLaunchTemplate(ctx, &launchTemplateOptions{
 		Cluster:        provisioner.Spec.Cluster,
-		UserData:       userData,
+		UserData:       p.getUserData(provisioner, constraints),
 		AMIID:          amiID,
 		SecurityGroups: securityGroups,
 	})
@@ -190,28 +197,14 @@ func (p *LaunchTemplateProvider) getSecurityGroupIds(ctx context.Context, provis
 	return securityGroupIds, nil
 }
 
-func (p *LaunchTemplateProvider) getUserData(ctx context.Context, provisioner *v1alpha3.Provisioner, constraints *Constraints) (string, error) {
+func (p *LaunchTemplateProvider) getUserData(provisioner *v1alpha3.Provisioner, constraints *Constraints) string {
+	t := template.Must(template.New("userData").Parse(bottlerocketUserData))
 	var userData bytes.Buffer
-	userData.WriteString(fmt.Sprintf("[settings.kubernetes]\napi-server = \"%s\"\n", provisioner.Spec.Cluster.Endpoint))
-	userData.WriteString(fmt.Sprintf("cluster-name = \"%s\"\n", *provisioner.Spec.Cluster.Name))
-	caBundle, err := provisioner.Spec.Cluster.GetCABundle(ctx)
-	if err != nil {
-		return "", fmt.Errorf("getting user data, %w", err)
-	}
-	if caBundle != nil {
-		userData.WriteString(fmt.Sprintf("cluster-certificate = \"%s\"\n", *caBundle))
-	}
-	if len(constraints.Labels) > 0 {
-		userData.WriteString("[settings.kubernetes.node-labels]\n")
-		for k, v := range constraints.Labels {
-			userData.WriteString(fmt.Sprintf("\"%s\" = \"%v\"\n", k, v))
-		}
-	}
-	if len(constraints.Taints) > 0 {
-		userData.WriteString("[settings.kubernetes.node-taints]\n")
-		for _, taint := range constraints.Taints {
-			userData.WriteString(fmt.Sprintf("\"%s\" = \"%s:%s\"\n", taint.Key, taint.Value, taint.Effect))
-		}
+	if err := t.Execute(&userData, struct {
+		Constraints *Constraints
+		Cluster     v1alpha3.Cluster
+	}{constraints, provisioner.Spec.Cluster}); err != nil {
+		panic(fmt.Sprintf("Parsing user data from %v, %v, %s", provisioner, constraints, err.Error()))
 	}
-	return base64.StdEncoding.EncodeToString(userData.Bytes()), nil
+	return base64.StdEncoding.EncodeToString(userData.Bytes())
 }

pkg/cloudprovider/aws/suite_test.go

@@ -335,7 +335,7 @@ var _ = Describe("Allocation", func() {
 				launchTemplate := input.LaunchTemplateConfigs[0].LaunchTemplateSpecification
 				Expect(*launchTemplate.LaunchTemplateName).To(Equal(pods[0].Spec.NodeSelector[LaunchTemplateNameLabel]))
 			})
-			It("should allow a pod to override the launch template name and use the default launch template version", func() {
+			It("should allow a pod to override the launch template id and use the default launch template version", func() {
 				// Setup
 				provisioner.Spec.Labels = map[string]string{LaunchTemplateNameLabel: randomdata.SillyName()}
 				ExpectCreated(env.Client, provisioner)
@@ -352,7 +352,7 @@ var _ = Describe("Allocation", func() {
 				Expect(*launchTemplate.LaunchTemplateName).To(Equal(pods[0].Spec.NodeSelector[LaunchTemplateNameLabel]))
 				Expect(*launchTemplate.Version).To(Equal(DefaultLaunchTemplateVersion))
 			})
-			It("should allow a pod to override the launch template name and use the provisioner's launch template version", func() {
+			It("should allow a pod to override the launch template id and use the provisioner's launch template version", func() {
 				// Setup
 				provisioner.Spec.Labels = map[string]string{
 					LaunchTemplateNameLabel: randomdata.SillyName(),
@@ -372,7 +372,7 @@ var _ = Describe("Allocation", func() {
 			})
 		})
 		Context("Subnets", func() {
-			It("should default to the cluster's subnets", func() {
+			It("should default to the clusters subnets", func() {
 				// Setup
 				provisioner.Spec.InstanceTypes = []string{"m5.large"} // limit instance type to simplify ConsistOf checks
 				ExpectCreated(env.Client, provisioner)

pkg/controllers/allocation/controller.go

@@ -172,7 +172,15 @@ func (c *Controller) provisionerFor(ctx context.Context, name types.NamespacedNa
 	if err := c.KubeClient.Get(ctx, name, provisioner); err != nil {
 		return nil, err
 	}
-	return provisioner, nil
+
+	// Hydrate provisioner with (dynamic) default values, which must not
+	//    be persisted into the original CRD as they might change with each reconciliation
+	//    loop iteration.
+	defaulted, err := provisioner.WithDynamicDefaults(ctx)
+	if err != nil {
+		return &defaulted, fmt.Errorf("setting dynamic default values, %w", err)
+	}
+	return &defaulted, nil
 }
 
 // podToProvisioner is a function handler to transform pod objs to provisioner reconcile requests

pkg/controllers/allocation/scheduling/scheduler.go

@@ -134,6 +134,7 @@ func (s *Scheduler) getDaemons(ctx context.Context, node *v1.Node) ([]*v1.Pod, e
 	return pods, nil
 }
 
+
 // IsSchedulable returns true if the pod can schedule to the node
 func IsSchedulable(pod *v1.Pod, node *v1.Node) bool {
 	// Tolerate Taints

website/content/en/docs/provisioner-crd.md

@@ -33,10 +33,10 @@ spec:
   # Provisioned nodes will have these labels
   labels:
     ##### AWS Specific #####
-    # Constrain node launch template ('$Default' version always used).
-    # If not specified, Karpenter will generate a Bottlerocket-
-    # based launch template dynamically.
-    node.k8s.aws/launch-template-name: "my-launch-template-name"
+    # Constrain node launch template, default="bottlerocket"
+    node.k8s.aws/launch-template-id: "bottlerocket-qwertyuiop"
+    # Constrain node launch template, default="$LATEST"
+    node.k8s.aws/launch-template-version: "my-special-version"
     # Constrain node capacity type, default="on-demand"
     node.k8s.aws/capacity-type: "spot"
 ```