diff --git a/.github/actions/e2e/install-karpenter/action.yaml b/.github/actions/e2e/install-karpenter/action.yaml index 4599e2dcca69..2af8061de567 100644 --- a/.github/actions/e2e/install-karpenter/action.yaml +++ b/.github/actions/e2e/install-karpenter/action.yaml @@ -49,35 +49,15 @@ runs: logout: true - name: install-karpenter shell: bash + env: + ECR_ACCOUNT_ID: ${{ inputs.ecr_account_id }} + ECR_REGION: ${{ inputs.ecr_region }} + ACCOUNT_ID: ${{ inputs.account_id }} + CLUSTER_NAME: ${{ inputs.cluster_name }} + K8S_VERSION: ${{ inputs.k8s_version }} + WEBHOOKS_ENABLED: ${{ inputs.webhooks_enabled }} run: | - aws eks update-kubeconfig --name "${{ inputs.cluster_name }}" - - # Parse minor version to determine whether to enable the webhooks - VERSION=${{ inputs.k8s_version }} - RELEASE_VERSION_MINOR="${VERSION#*.}" - - helm upgrade --install karpenter oci://${{ inputs.ecr_account_id }}.dkr.ecr.${{ inputs.ecr_region }}.amazonaws.com/karpenter/snapshot/karpenter \ - -n kube-system \ - --version "v0-$(git rev-parse HEAD)" \ - --set serviceAccount.annotations."eks\.amazonaws\.com/role-arn"="arn:aws:iam::${{ inputs.account_id }}:role/karpenter-irsa-${{ inputs.cluster_name }}" \ - --set webhook.enabled=${{ inputs.webhooks_enabled }} \ - --set settings.clusterName="${{ inputs.cluster_name }}" \ - --set settings.interruptionQueue="${{ inputs.cluster_name }}" \ - --set controller.resources.requests.cpu=3 \ - --set controller.resources.requests.memory=3Gi \ - --set controller.resources.limits.cpu=3 \ - --set controller.resources.limits.memory=3Gi \ - --set serviceMonitor.enabled=true \ - --set serviceMonitor.additionalLabels.scrape=enabled \ - --set "serviceMonitor.endpointConfig.relabelings[0].targetLabel=clusterName" \ - --set "serviceMonitor.endpointConfig.relabelings[0].replacement=${{ inputs.cluster_name }}" \ - --set "serviceMonitor.endpointConfig.relabelings[1].targetLabel=gitRef" \ - --set "serviceMonitor.endpointConfig.relabelings[1].replacement=$(git rev-parse HEAD)" \ - --set "serviceMonitor.endpointConfig.relabelings[2].targetLabel=mostRecentTag" \ - --set "serviceMonitor.endpointConfig.relabelings[2].replacement=$(git describe --abbrev=0 --tags)" \ - --set "serviceMonitor.endpointConfig.relabelings[3].targetLabel=commitsAfterTag" \ - --set "serviceMonitor.endpointConfig.relabelings[3].replacement=\"$(git describe --tags | cut -d '-' -f 2)\"" \ - --wait + ./test/hack/e2e_scripts/install_karpenter.sh - name: diff-karpenter shell: bash run: | diff --git a/charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml b/charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml index db3073731fe3..981f9c83a39e 100644 --- a/charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml +++ b/charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml @@ -19,6 +19,7 @@ spec: singular: ec2nodeclass scope: Cluster versions: +{{- if .Values.webhook.enabled }} - additionalPrinterColumns: - jsonPath: .status.conditions[?(@.type=="Ready")].status name: Ready @@ -735,6 +736,7 @@ spec: storage: false subresources: status: {} +{{- end }} - name: v1beta1 schema: openAPIV3Schema: diff --git a/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml b/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml index d03f99572890..9a00f213e91e 100644 --- a/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml +++ b/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml @@ -16,6 +16,7 @@ spec: singular: nodeclaim scope: Cluster versions: +{{- if .Values.webhook.enabled }} - additionalPrinterColumns: - jsonPath: .metadata.labels.node\.kubernetes\.io/instance-type name: Type @@ -379,6 +380,7 @@ spec: storage: false subresources: status: {} +{{- end }} - additionalPrinterColumns: - jsonPath: .metadata.labels.node\.kubernetes\.io/instance-type name: Type diff --git a/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml b/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml index f6ea6d7a46c5..fd0ce7eeeef4 100644 --- a/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml +++ b/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml @@ -16,6 +16,7 @@ spec: singular: nodepool scope: Cluster versions: +{{- if .Values.webhook.enabled }} - additionalPrinterColumns: - jsonPath: .spec.template.spec.nodeClassRef.name name: NodeClass @@ -502,6 +503,7 @@ spec: storage: false subresources: status: {} +{{- end }} - additionalPrinterColumns: - jsonPath: .spec.template.spec.nodeClassRef.name name: NodeClass diff --git a/hack/mutation/conversion_webhook_injection.sh b/hack/mutation/conversion_webhook_injection.sh index ef57c2e6fd12..ec878cfdcbbc 100755 --- a/hack/mutation/conversion_webhook_injection.sh +++ b/hack/mutation/conversion_webhook_injection.sh @@ -6,54 +6,12 @@ yq eval '.spec.conversion = {"strategy": "Webhook", "webhook": {"conversionRevie yq eval '.spec.conversion = {"strategy": "Webhook", "webhook": {"conversionReviewVersions": ["v1beta1", "v1"], "clientConfig": {"service": {"name": "karpenter", "namespace": "kube-system", "port": 8443}}}}' -i pkg/apis/crds/karpenter.sh_nodepools.yaml # Update to the karpenter-crd charts - -# Remove the copied over conversion stanzas from CRD spec +# Remove the copied conversion stanzas from CRD specs yq eval 'del(.spec.conversion)' -i charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml yq eval 'del(.spec.conversion)' -i charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml yq eval 'del(.spec.conversion)' -i charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml -# Add the conversion stanza template to the CRD spec to enable conversion via webhook -echo "{{- if .Values.webhook.enabled }} - conversion: - strategy: Webhook - webhook: - conversionReviewVersions: - - v1beta1 - - v1 - clientConfig: - service: - name: {{ .Values.webhook.serviceName }} - namespace: {{ .Values.webhook.serviceNamespace | default .Release.Namespace }} - port: {{ .Values.webhook.port }} -{{- end }} -" >> charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml - -echo "{{- if .Values.webhook.enabled }} - conversion: - strategy: Webhook - webhook: - conversionReviewVersions: - - v1beta1 - - v1 - clientConfig: - service: - name: {{ .Values.webhook.serviceName }} - namespace: {{ .Values.webhook.serviceNamespace | default .Release.Namespace }} - port: {{ .Values.webhook.port }} -{{- end }} -" >> charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml - -echo "{{- if .Values.webhook.enabled }} - conversion: - strategy: Webhook - webhook: - conversionReviewVersions: - - v1beta1 - - v1 - clientConfig: - service: - name: {{ .Values.webhook.serviceName }} - namespace: {{ .Values.webhook.serviceNamespace | default .Release.Namespace }} - port: {{ .Values.webhook.port }} -{{- end }} -" >> charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml \ No newline at end of file +# Template the v1 version and the conversion strategy of the spec +hack/mutation/ec2nodeclasses.sh +hack/mutation/nodepools.sh +hack/mutation/nodeclaims.sh diff --git a/hack/mutation/ec2nodeclasses.sh b/hack/mutation/ec2nodeclasses.sh new file mode 100755 index 000000000000..5d442b182b67 --- /dev/null +++ b/hack/mutation/ec2nodeclasses.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +VERSION_START="$(cat charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml | yq '.spec.versions.[0] | line')" +VERSION_END="$(cat charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml | yq '.spec.versions.[1] | line')" +VERSION_END=$(($VERSION_END+1)) +TEMP=$(mktemp) + +cat charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml | awk -v n=$VERSION_START 'NR==n {sub(/$/,"\n{{- if .Values.webhook.enabled }}")} 1' \ +| awk -v n=$VERSION_END 'NR==n {sub(/$/,"\n{{- end }}")} 1' > $TEMP + +cat $TEMP > charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml + +echo "{{- if .Values.webhook.enabled }} + conversion: + strategy: Webhook + webhook: + conversionReviewVersions: + - v1beta1 + - v1 + clientConfig: + service: + name: {{ .Values.webhook.serviceName }} + namespace: {{ .Values.webhook.serviceNamespace | default .Release.Namespace }} + port: {{ .Values.webhook.port }} +{{- end }} +" >> charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml \ No newline at end of file diff --git a/hack/mutation/nodeclaims.sh b/hack/mutation/nodeclaims.sh new file mode 100755 index 000000000000..77afccdf8a47 --- /dev/null +++ b/hack/mutation/nodeclaims.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +VERSION_START="$(cat charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml | yq '.spec.versions.[0] | line')" +VERSION_END="$(cat charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml | yq '.spec.versions.[1] | line')" +VERSION_END=$(($VERSION_END+1)) +TEMP=$(mktemp) + +cat charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml | awk -v n=$VERSION_START 'NR==n {sub(/$/,"\n{{- if .Values.webhook.enabled }}")} 1' \ +| awk -v n=$VERSION_END 'NR==n {sub(/$/,"\n{{- end }}")} 1' > $TEMP + +cat $TEMP > charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml + +echo "{{- if .Values.webhook.enabled }} + conversion: + strategy: Webhook + webhook: + conversionReviewVersions: + - v1beta1 + - v1 + clientConfig: + service: + name: {{ .Values.webhook.serviceName }} + namespace: {{ .Values.webhook.serviceNamespace | default .Release.Namespace }} + port: {{ .Values.webhook.port }} +{{- end }} +" >> charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml \ No newline at end of file diff --git a/hack/mutation/nodepools.sh b/hack/mutation/nodepools.sh new file mode 100755 index 000000000000..9ad1a6d279fa --- /dev/null +++ b/hack/mutation/nodepools.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +VERSION_START="$(cat charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml | yq '.spec.versions.[0] | line')" +VERSION_END="$(cat charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml | yq '.spec.versions.[1] | line')" +VERSION_END=$(($VERSION_END+1)) +TEMP=$(mktemp) + +cat charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml | awk -v n=$VERSION_START 'NR==n {sub(/$/,"\n{{- if .Values.webhook.enabled }}")} 1' \ +| awk -v n=$VERSION_END 'NR==n {sub(/$/,"\n{{- end }}")} 1' > $TEMP + +cat $TEMP > charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml + +echo "{{- if .Values.webhook.enabled }} + conversion: + strategy: Webhook + webhook: + conversionReviewVersions: + - v1beta1 + - v1 + clientConfig: + service: + name: {{ .Values.webhook.serviceName }} + namespace: {{ .Values.webhook.serviceNamespace | default .Release.Namespace }} + port: {{ .Values.webhook.port }} +{{- end }} +" >> charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml \ No newline at end of file diff --git a/test/hack/e2e_scripts/install_karpenter.sh b/test/hack/e2e_scripts/install_karpenter.sh new file mode 100755 index 000000000000..e3395e0224d2 --- /dev/null +++ b/test/hack/e2e_scripts/install_karpenter.sh @@ -0,0 +1,37 @@ +aws eks update-kubeconfig --name "$CLUSTER_NAME" + +# First, conditionally install the webhook stanza and CRDs +if (( "$WEBHOOKS_ENABLED" == false )); then +helm upgrade --install karpenter-crd oci://$ECR_ACCOUNT_ID.dkr.ecr.$ECR_REGION.amazonaws.com/karpenter/snapshot/karpenter-crd \ + --namespace kube-system \ + --version "0-$(git rev-parse HEAD)" \ + --set webhook.enabled=${WEBHOOKS_ENABLED} \ + --wait +fi + +CHART="oci://$ECR_ACCOUNT_ID.dkr.ecr.$ECR_REGION.amazonaws.com/karpenter/snapshot/karpenter" +ADDITIONAL_FLAGS="" + +# Remove service account annotation when dropping support for 1.23 +helm upgrade --install karpenter "${CHART}" \ + -n kube-system \ + --version "v0-$(git rev-parse HEAD)" \ + --set serviceAccount.annotations."eks\.amazonaws\.com/role-arn"="arn:aws:iam::$ACCOUNT_ID:role/karpenter-irsa-$CLUSTER_NAME" \ + --set webhook.enabled=${WEBHOOKS_ENABLED} \ + --set settings.clusterName="$CLUSTER_NAME" \ + --set settings.interruptionQueue="$CLUSTER_NAME" \ + --set controller.resources.requests.cpu=3 \ + --set controller.resources.requests.memory=3Gi \ + --set controller.resources.limits.cpu=3 \ + --set controller.resources.limits.memory=3Gi \ + --set serviceMonitor.enabled=true \ + --set serviceMonitor.additionalLabels.scrape=enabled \ + --set "serviceMonitor.endpointConfig.relabelings[0].targetLabel=clusterName" \ + --set "serviceMonitor.endpointConfig.relabelings[0].replacement=$CLUSTER_NAME" \ + --set "serviceMonitor.endpointConfig.relabelings[1].targetLabel=gitRef" \ + --set "serviceMonitor.endpointConfig.relabelings[1].replacement=$(git rev-parse HEAD)" \ + --set "serviceMonitor.endpointConfig.relabelings[2].targetLabel=mostRecentTag" \ + --set "serviceMonitor.endpointConfig.relabelings[2].replacement=$(git describe --abbrev=0 --tags)" \ + --set "serviceMonitor.endpointConfig.relabelings[3].targetLabel=commitsAfterTag" \ + --set "serviceMonitor.endpointConfig.relabelings[3].replacement=\"$(git describe --tags | cut -d '-' -f 2)\"" \ + --wait \ No newline at end of file