From 4c6d3ff8f6715ccf7010e674e389576ba489da74 Mon Sep 17 00:00:00 2001 From: Ellis Tarn Date: Thu, 11 Nov 2021 09:33:04 -0800 Subject: [PATCH] Fixes an issue with webhook crashlooping (#794) --- Makefile | 1 + .../templates/webhook/deployment.yaml | 4 ++++ charts/karpenter/values.yaml | 3 +-- cmd/controller/main.go | 17 +------------- cmd/webhook/main.go | 17 +++++--------- pkg/utils/options/options.go | 23 +++++++++++++++++-- 6 files changed, 34 insertions(+), 31 deletions(-) diff --git a/Makefile b/Makefile index e9b63e37871d..8005e8bed1a5 100644 --- a/Makefile +++ b/Makefile @@ -59,6 +59,7 @@ apply: ## Deploy the controller into your ~/.kube/config cluster delete: ## Delete the controller from your ~/.kube/config cluster helm template karpenter charts/karpenter --namespace karpenter \ --set serviceAccount.create=false \ + --set defaultProvisioner.create=false \ | kubectl delete -f - codegen: ## Generate code. Must be run if changes are made to ./pkg/apis/... diff --git a/charts/karpenter/templates/webhook/deployment.yaml b/charts/karpenter/templates/webhook/deployment.yaml index cb096da6ea63..62b920adf0ad 100644 --- a/charts/karpenter/templates/webhook/deployment.yaml +++ b/charts/karpenter/templates/webhook/deployment.yaml @@ -47,6 +47,10 @@ spec: scheme: HTTPS port: 8443 env: + - name: CLUSTER_NAME + value: {{ .Values.controller.clusterName }} + - name: CLUSTER_ENDPOINT + value: {{ .Values.controller.clusterEndpoint }} - name: SYSTEM_NAMESPACE valueFrom: fieldRef: diff --git a/charts/karpenter/values.yaml b/charts/karpenter/values.yaml index cb02636b5372..f9c3dcea03f1 100644 --- a/charts/karpenter/values.yaml +++ b/charts/karpenter/values.yaml @@ -23,9 +23,8 @@ webhook: defaultProvisioner: create: true ttlSecondsAfterEmpty: 300 - #ttlSecondsUntilExpired: 86400 + #ttlSecondsUntilExpired: 86400 provider: {} requirements: [] taints: [] labels: {} - diff --git a/cmd/controller/main.go b/cmd/controller/main.go index 08d5430b3b5b..005e01db72b4 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -16,7 +16,6 @@ package main import ( "context" - "flag" "fmt" "github.com/awslabs/karpenter/pkg/apis" @@ -27,7 +26,6 @@ import ( "github.com/awslabs/karpenter/pkg/controllers/metrics" "github.com/awslabs/karpenter/pkg/controllers/node" "github.com/awslabs/karpenter/pkg/controllers/termination" - "github.com/awslabs/karpenter/pkg/utils/env" "github.com/awslabs/karpenter/pkg/utils/options" "github.com/awslabs/karpenter/pkg/utils/restconfig" "github.com/go-logr/zapr" @@ -49,7 +47,7 @@ import ( var ( scheme = runtime.NewScheme() - opts = options.Options{} + opts = options.MustParse() component = "controller" ) @@ -59,14 +57,6 @@ func init() { } func main() { - flag.StringVar(&opts.ClusterName, "cluster-name", env.WithDefaultString("CLUSTER_NAME", ""), "The kubernetes cluster name for resource discovery") - flag.StringVar(&opts.ClusterEndpoint, "cluster-endpoint", env.WithDefaultString("CLUSTER_ENDPOINT", ""), "The external kubernetes cluster endpoint for new nodes to connect with") - flag.IntVar(&opts.MetricsPort, "metrics-port", env.WithDefaultInt("METRICS_PORT", 8080), "The port the metric endpoint binds to for operating metrics about the controller itself") - flag.IntVar(&opts.HealthProbePort, "health-probe-port", env.WithDefaultInt("HEALTH_PROBE_PORT", 8081), "The port the health probe endpoint binds to for reporting controller health") - flag.IntVar(&opts.KubeClientQPS, "kube-client-qps", env.WithDefaultInt("KUBE_CLIENT_QPS", 200), "The smoothed rate of qps to kube-apiserver") - flag.IntVar(&opts.KubeClientBurst, "kube-client-burst", env.WithDefaultInt("KUBE_CLIENT_BURST", 300), "The maximum allowed burst of queries to the kube-apiserver") - flag.Parse() - if err := opts.Validate(); err != nil { panic(fmt.Sprintf("Input parameter validation failed, %s", err.Error())) } @@ -77,12 +67,7 @@ func main() { // Set up logger and watch for changes to log level ctx := LoggingContextOrDie(config, clientSet) - - // Put REST config in context, as it can be used by arbitrary - // parts of the code base ctx = restconfig.Inject(ctx, config) - - // Put CLI args into context for access across code base ctx = options.Inject(ctx, opts) // Set up controller runtime controller diff --git a/cmd/webhook/main.go b/cmd/webhook/main.go index 1283cf00fb15..7f134db8abdb 100644 --- a/cmd/webhook/main.go +++ b/cmd/webhook/main.go @@ -16,11 +16,11 @@ package main import ( "context" - "flag" "github.com/awslabs/karpenter/pkg/apis" "github.com/awslabs/karpenter/pkg/cloudprovider" "github.com/awslabs/karpenter/pkg/cloudprovider/registry" + "github.com/awslabs/karpenter/pkg/utils/options" "k8s.io/client-go/kubernetes" "knative.dev/pkg/configmap" "knative.dev/pkg/controller" @@ -37,20 +37,13 @@ import ( ) var ( - options = Options{} + opts = options.MustParse() ) -type Options struct { - Port int -} - func main() { - flag.IntVar(&options.Port, "port", 8443, "The port the webhook endpoint binds to for validation and mutation of resources") - flag.Parse() - config := injection.ParseAndGetRESTConfigOrDie() ctx := webhook.WithOptions(injection.WithNamespaceScope(signals.NewContext(), system.Namespace()), webhook.Options{ - Port: options.Port, + Port: opts.WebhookPort, ServiceName: "karpenter-webhook", SecretName: "karpenter-webhook-cert", }) @@ -97,4 +90,6 @@ func newConfigValidationController(ctx context.Context, cmw configmap.Watcher) * ) } -func InjectContext(ctx context.Context) context.Context { return ctx } +func InjectContext(ctx context.Context) context.Context { + return options.Inject(ctx, opts) +} diff --git a/pkg/utils/options/options.go b/pkg/utils/options/options.go index ceea8c6195de..d929c1d3524b 100644 --- a/pkg/utils/options/options.go +++ b/pkg/utils/options/options.go @@ -16,18 +16,37 @@ package options import ( "context" + "flag" "fmt" "net/url" + "github.com/awslabs/karpenter/pkg/utils/env" "go.uber.org/multierr" ) +func MustParse() Options { + opts := Options{} + flag.StringVar(&opts.ClusterName, "cluster-name", env.WithDefaultString("CLUSTER_NAME", ""), "The kubernetes cluster name for resource discovery") + flag.StringVar(&opts.ClusterEndpoint, "cluster-endpoint", env.WithDefaultString("CLUSTER_ENDPOINT", ""), "The external kubernetes cluster endpoint for new nodes to connect with") + flag.IntVar(&opts.MetricsPort, "metrics-port", env.WithDefaultInt("METRICS_PORT", 8080), "The port the metric endpoint binds to for operating metrics about the controller itself") + flag.IntVar(&opts.HealthProbePort, "health-probe-port", env.WithDefaultInt("HEALTH_PROBE_PORT", 8081), "The port the health probe endpoint binds to for reporting controller health") + flag.IntVar(&opts.WebhookPort, "port", 8443, "The port the webhook endpoint binds to for validation and mutation of resources") + flag.IntVar(&opts.KubeClientQPS, "kube-client-qps", env.WithDefaultInt("KUBE_CLIENT_QPS", 200), "The smoothed rate of qps to kube-apiserver") + flag.IntVar(&opts.KubeClientBurst, "kube-client-burst", env.WithDefaultInt("KUBE_CLIENT_BURST", 300), "The maximum allowed burst of queries to the kube-apiserver") + flag.Parse() + if err := opts.Validate(); err != nil { + panic(err) + } + return opts +} + // Options for running this binary type Options struct { ClusterName string ClusterEndpoint string MetricsPort int HealthProbePort int + WebhookPort int KubeClientQPS int KubeClientBurst int } @@ -38,8 +57,8 @@ func Get(ctx context.Context) Options { return ctx.Value(optionsKey{}).(Options) } -func Inject(ctx context.Context, options Options) context.Context { - return context.WithValue(ctx, optionsKey{}, options) +func Inject(ctx context.Context, opts Options) context.Context { + return context.WithValue(ctx, optionsKey{}, opts) } func (o Options) Validate() (err error) {