diff --git a/.github/workflows/readme.yml b/.github/workflows/readme.yml index 74f7f2a..0661a59 100644 --- a/.github/workflows/readme.yml +++ b/.github/workflows/readme.yml @@ -67,7 +67,7 @@ jobs: - name: Generate TF Docs uses: terraform-docs/gh-actions@v1.0.0 with: - working-dir: addons/aws-ebs-csi-driver,addons/aws-efs-csi-driver,addons/aws-load-balancer-controller,addons/aws-node-termination-handler,addons/calico-tigera,addons/cluster-autoscaler,addons/external-secrets,addons/fluent-bit,addons/helm,addons/ingress-nginx,addons/istio-ingress,addons/karpenter,addons/kiali-server,addons/kubeclarity,addons/metrics-server,addons/nri-bundle,addons/velero,addons/kube-state-metrics + working-dir: addons/aws-ebs-csi-driver,addons/aws-efs-csi-driver,addons/aws-load-balancer-controller,addons/aws-node-termination-handler,addons/calico-tigera,addons/cluster-autoscaler,addons/external-secrets,addons/fluent-bit,addons/helm,addons/ingress-nginx,addons/istio-ingress,addons/karpenter,addons/kiali-server,addons/kubeclarity,addons/metrics-server,addons/nri-bundle,addons/velero,addons/kube-state-metrics,addons/keda git-push: true template: |- diff --git a/_examples/basic/config/keda/override-keda.yaml b/_examples/basic/config/keda/override-keda.yaml new file mode 100644 index 0000000..d169381 --- /dev/null +++ b/_examples/basic/config/keda/override-keda.yaml @@ -0,0 +1,9 @@ +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "eks.amazonaws.com/nodegroup" + operator: In + values: + - "critical" \ No newline at end of file diff --git a/_examples/basic/config/keda/strategy_example.yaml b/_examples/basic/config/keda/strategy_example.yaml new file mode 100644 index 0000000..7e1a12b --- /dev/null +++ b/_examples/basic/config/keda/strategy_example.yaml @@ -0,0 +1,36 @@ +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: nginx-deployment + namespace: nginx +spec: + scaleTargetRef: + apiVersion: apps/v1 # Optional. Default: apps/v1 + kind: Deployment # Optional. Default: Deployment + name: nginx # Mandatory. Must be in the same namespace as the ScaledObject + pollingInterval: 5 # Optional. Default: 5 seconds + cooldownPeriod: 20 # Optional. Default: 300 seconds + minReplicaCount: 1 # Optional. Default: 0 + maxReplicaCount: 10 # Optional. Default: 100 + fallback: # Optional. Section to specify fallback options + failureThreshold: 3 # Mandatory if fallback section is included + replicas: 1 # Mandatory if fallback section is included + advanced: # Optional. Section to specify advanced options + restoreToOriginalReplicaCount: true # Optional. Default: false + horizontalPodAutoscalerConfig: # Optional. Section to specify HPA related options + name: keda-hpa-nginx # Optional. Default: keda-hpa-{scaled-object-name} + behavior: # Optional. Use to modify HPA's scaling behavior + scaleDown: + stabilizationWindowSeconds: 60 + policies: + - type: Percent + value: 100 + periodSeconds: 15 + triggers: + - type: cron + metadata: + # Required + timezone: Asia/Kolkata # The acceptable values would be a value from the IANA Time Zone Database. + start: "12 00 * * *" + end: "15 00 * * *" + desiredReplicas: "5" \ No newline at end of file diff --git a/_examples/basic/main.tf b/_examples/basic/main.tf index bf6de50..bc54fc6 100644 --- a/_examples/basic/main.tf +++ b/_examples/basic/main.tf @@ -172,6 +172,7 @@ module "addons" { ingress_nginx = true fluent_bit = true velero = true + keda = true # -- Addons with mandatory variable istio_ingress = true diff --git a/_examples/complete/config/keda/override-keda.yaml b/_examples/complete/config/keda/override-keda.yaml new file mode 100644 index 0000000..d169381 --- /dev/null +++ b/_examples/complete/config/keda/override-keda.yaml @@ -0,0 +1,9 @@ +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "eks.amazonaws.com/nodegroup" + operator: In + values: + - "critical" \ No newline at end of file diff --git a/_examples/complete/config/keda/strategy_example.yaml b/_examples/complete/config/keda/strategy_example.yaml new file mode 100644 index 0000000..7e1a12b --- /dev/null +++ b/_examples/complete/config/keda/strategy_example.yaml @@ -0,0 +1,36 @@ +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: nginx-deployment + namespace: nginx +spec: + scaleTargetRef: + apiVersion: apps/v1 # Optional. Default: apps/v1 + kind: Deployment # Optional. Default: Deployment + name: nginx # Mandatory. Must be in the same namespace as the ScaledObject + pollingInterval: 5 # Optional. Default: 5 seconds + cooldownPeriod: 20 # Optional. Default: 300 seconds + minReplicaCount: 1 # Optional. Default: 0 + maxReplicaCount: 10 # Optional. Default: 100 + fallback: # Optional. Section to specify fallback options + failureThreshold: 3 # Mandatory if fallback section is included + replicas: 1 # Mandatory if fallback section is included + advanced: # Optional. Section to specify advanced options + restoreToOriginalReplicaCount: true # Optional. Default: false + horizontalPodAutoscalerConfig: # Optional. Section to specify HPA related options + name: keda-hpa-nginx # Optional. Default: keda-hpa-{scaled-object-name} + behavior: # Optional. Use to modify HPA's scaling behavior + scaleDown: + stabilizationWindowSeconds: 60 + policies: + - type: Percent + value: 100 + periodSeconds: 15 + triggers: + - type: cron + metadata: + # Required + timezone: Asia/Kolkata # The acceptable values would be a value from the IANA Time Zone Database. + start: "12 00 * * *" + end: "15 00 * * *" + desiredReplicas: "5" \ No newline at end of file diff --git a/_examples/complete/main.tf b/_examples/complete/main.tf index be510b6..3188165 100644 --- a/_examples/complete/main.tf +++ b/_examples/complete/main.tf @@ -167,6 +167,7 @@ module "addons" { ingress_nginx = true fluent_bit = true velero = true + keda = true # -- Addons with mandatory variable istio_ingress = true @@ -194,6 +195,7 @@ module "addons" { velero_helm_config = { values = [file("./config/override-velero.yaml")] } new_relic_helm_config = { values = [file("./config/override-new-relic.yaml")] } kube_state_metrics_helm_config = { values = [file("./config/override-kube-state-matrics.yaml")] } + keda_helm_config = { values = [file("./config/keda/override-keda.yaml")] } # -- Override Helm Release attributes metrics_server_extra_configs = var.metrics_server_extra_configs @@ -213,6 +215,7 @@ module "addons" { velero_extra_configs = var.velero_extra_configs new_relic_extra_configs = var.new_relic_extra_configs kube_state_metrics_extra_configs = var.kube_state_metrics_extra_configs + keda_extra_configs = var.keda_extra_configs # -- Custom IAM Policy Json for Addon's ServiceAccount cluster_autoscaler_iampolicy_json_content = file("./custom-iam-policies/cluster-autoscaler.json") diff --git a/_examples/complete/variables.tf b/_examples/complete/variables.tf index 9882ed3..1c0c911 100644 --- a/_examples/complete/variables.tf +++ b/_examples/complete/variables.tf @@ -134,4 +134,9 @@ variable "new_relic_extra_configs" { variable "kube_state_metrics_extra_configs" { type = any default = {} +} + +variable "keda_extra_configs" { + type = any + default = {} } \ No newline at end of file diff --git a/addons/keda/README.md b/addons/keda/README.md new file mode 100644 index 0000000..ace0d79 --- /dev/null +++ b/addons/keda/README.md @@ -0,0 +1,70 @@ +# Keda Helm Chart + +KEDA allows for fine grained autoscaling (including to/from zero) for event driven Kubernetes workloads. + +## Installation +Below terraform script shows how to use Keda Terraform Addon, A complete example is also given [here](https://github.com/clouddrove/terraform-helm-eks-addons/blob/master/_examples/complete/main.tf). +```bash +module "addons" { + source = "clouddrove/eks-addons/aws" + version = "0.0.9" + + depends_on = [module.eks.cluster_id] + eks_cluster_name = module.eks.cluster_name + + keda = true +} +``` + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0.0 | +| [aws](#requirement\_aws) | >= 3.72 | +| [kubernetes](#requirement\_kubernetes) | >= 2.10 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 3.72 | +| [kubernetes](#provider\_kubernetes) | >= 2.10 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [helm\_addon](#module\_helm\_addon) | ../helm | n/a | + +## Resources + +| Name | Type | +|------|------| +| [aws_iam_policy.policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [kubernetes_namespace_v1.this](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace_v1) | resource | +| [aws_eks_cluster.eks_cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [account\_id](#input\_account\_id) | n/a | `string` | `""` | no | +| [addon\_context](#input\_addon\_context) | Input configuration for the addon | 
object({
    aws_caller_identity_account_id = string
    aws_caller_identity_arn        = string
    aws_eks_cluster_endpoint       = string
    aws_partition_id               = string
    aws_region_name                = string
    eks_cluster_id                 = string
    eks_oidc_issuer_url            = string
    eks_oidc_provider_arn          = string
    tags                           = map(string)
  })
| n/a | yes | +| [aws\_load\_balancer\_controller\_extra\_configs](#input\_keda\_configs) | Override attributes of helm\_release terraform resource | `any` | `{}` | no | +| [eks\_cluster\_name](#input\_eks\_cluster\_name) | n/a | `string` | `""` | no | +| [helm\_config](#input\_helm\_config) | Helm provider config for Keda | `any` | `{}` | no | +| [iampolicy\_json\_content](#input\_iampolicy\_json\_content) | Custom IAM Policy for Keda IRSA | `string` | `null` | no | +| [manage\_via\_gitops](#input\_manage\_via\_gitops) | Determines if the add-on should be managed via GitOps | `bool` | `false` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [chart\_version](#output\_chart\_version) | n/a | +| [iam\_policy](#output\_iam\_policy) | n/a | +| [namespace](#output\_namespace) | n/a | +| [repository](#output\_repository) | n/a | +| [service\_account](#output\_service\_account) | n/a | + diff --git a/addons/keda/config/keda.yaml b/addons/keda/config/keda.yaml new file mode 100644 index 0000000..82d6fb8 --- /dev/null +++ b/addons/keda/config/keda.yaml @@ -0,0 +1,634 @@ +# Default values for keda. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +image: + keda: + # -- Image name of KEDA operator + repository: ghcr.io/kedacore/keda + # -- Image tag of KEDA operator. Optional, given app version of Helm chart is used by default + tag: "" + metricsApiServer: + # -- Image name of KEDA Metrics API Server + repository: ghcr.io/kedacore/keda-metrics-apiserver + # -- Image tag of KEDA Metrics API Server. Optional, given app version of Helm chart is used by default + tag: "" + webhooks: + # -- Image name of KEDA admission-webhooks + repository: ghcr.io/kedacore/keda-admission-webhooks + # -- Image tag of KEDA admission-webhooks . Optional, given app version of Helm chart is used by default + tag: "" + # -- Image pullPolicy for all KEDA components + pullPolicy: Always + +# -- Kubernetes cluster domain +clusterDomain: cluster.local + +crds: + # -- Defines whether the KEDA CRDs have to be installed or not. + install: true + +# -- Defines Kubernetes namespaces to watch to scale their workloads. Default watches all namespaces +watchNamespace: "" + +# -- Name of secret to use to pull images to use to pull Docker images +imagePullSecrets: [] + +operator: + # -- Name of the KEDA operator + name: keda-operator + # -- Capability to configure the number of replicas for KEDA operator. + # While you can run more replicas of our operator, only one operator instance will be the leader and serving traffic. + # You can run multiple replicas, but they will not improve the performance of KEDA, it could only reduce downtime during a failover. + # Learn more in [our documentation](https://keda.sh/docs/latest/operate/cluster/#high-availability). + replicaCount: 1 + # -- [Affinity] for pod scheduling for KEDA operator. Takes precedence over the `affinity` field + affinity: {} + # podAntiAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # - labelSelector: + # matchExpressions: + # - key: app + # operator: In + # values: + # - keda-operator + # topologyKey: "kubernetes.io/hostname" + +metricsServer: + # -- Capability to configure the number of replicas for KEDA metric server. + # While you can run more replicas of our metric server, only one instance will used and serve traffic. + # You can run multiple replicas, but they will not improve the performance of KEDA, it could only reduce downtime during a failover. + # Learn more in [our documentation](https://keda.sh/docs/latest/operate/cluster/#high-availability). + replicaCount: 1 + # use ClusterFirstWithHostNet if `useHostNetwork: true` https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy + # -- Defined the DNS policy for the metric server + dnsPolicy: ClusterFirst + # -- Enable metric server to use host network + useHostNetwork: false + # -- [Affinity] for pod scheduling for Metrics API Server. Takes precedence over the `affinity` field + affinity: {} + # podAntiAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # - labelSelector: + # matchExpressions: + # - key: app + # operator: In + # values: + # - keda-operator-metrics-apiserver + # topologyKey: "kubernetes.io/hostname" + +webhooks: + # -- Enable admission webhooks (this feature option will be removed in v2.12) + enabled: true + # -- Port number to use for KEDA admission webhooks. Default is 9443. + port: "" + # -- Port number to use for KEDA admission webhooks health probe + healthProbePort: 8081 + # -- Enable webhook to use host network, this is required on EKS with custom CNI + useHostNetwork: false + # -- Name of the KEDA admission webhooks + name: keda-admission-webhooks + # -- Capability to configure the number of replicas for KEDA admission webhooks + replicaCount: 1 + # -- [Affinity] for pod scheduling for KEDA admission webhooks. Takes precedence over the `affinity` field + affinity: {} + # podAntiAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # - labelSelector: + # matchExpressions: + # - key: app + # operator: In + # values: + # - keda-operator + # topologyKey: "kubernetes.io/hostname" + + # -- [Failure policy](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#failure-policy) to use with KEDA admission webhooks + failurePolicy: Ignore + +upgradeStrategy: + # -- Capability to configure [Deployment upgrade strategy] for operator + operator: {} + # type: RollingUpdate + # rollingUpdate: + # maxUnavailable: 1 + # maxSurge: 1 + + # -- Capability to configure [Deployment upgrade strategy] for Metrics Api Server + metricsApiServer: {} + # type: RollingUpdate + # rollingUpdate: + # maxUnavailable: 1 + # maxSurge: 1 + + # -- Capability to configure [Deployment upgrade strategy] for Admission webhooks + webhooks: {} + # type: RollingUpdate + # rollingUpdate: + # maxUnavailable: 1 + # maxSurge: 1 + +podDisruptionBudget: + # -- Capability to configure [Pod Disruption Budget] + operator: {} + # minAvailable: 1 + # maxUnavailable: 1 + + # -- Capability to configure [Pod Disruption Budget] + metricServer: {} + # minAvailable: 1 + # maxUnavailable: 1 + + # -- Capability to configure [Pod Disruption Budget] + webhooks: {} + # minAvailable: 1 + # maxUnavailable: 1 + +# -- Custom labels to add into metadata +additionalLabels: + {} + # foo: bar + +# -- Custom annotations to add into metadata +additionalAnnotations: + {} + # foo: bar + +podAnnotations: + # -- Pod annotations for KEDA operator + keda: {} + # -- Pod annotations for KEDA Metrics Adapter + metricsAdapter: {} + # -- Pod annotations for KEDA Admission webhooks + webhooks: {} +podLabels: + # -- Pod labels for KEDA operator + keda: {} + # -- Pod labels for KEDA Metrics Adapter + metricsAdapter: {} + # -- Pod labels for KEDA Admission webhooks + webhooks: {} + +rbac: + # -- Specifies whether RBAC should be used + create: true + # -- Specifies whether RBAC for CRDs should be [aggregated](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#aggregated-clusterroles) to default roles (view, edit, admin) + aggregateToDefaultRoles: false + +serviceAccount: + # -- Specifies whether a service account should be created + create: true + # -- The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: keda-operator + # -- Specifies whether a service account should automount API-Credentials + automountServiceAccountToken: true + # -- Annotations to add to the service account + annotations: {} + +podIdentity: + activeDirectory: + # Set to the value of the Azure Active Directory Pod Identity + # See https://keda.sh/docs/concepts/authentication/#azure-pod-identity + # This will be set as a label on the KEDA Pod(s) + # -- Identity in Azure Active Directory to use for Azure pod identity + identity: "" + azureWorkload: + # -- Set to true to enable Azure Workload Identity usage. + # See https://keda.sh/docs/concepts/authentication/#azure-workload-identity + # This will be set as a label on the KEDA service account. + enabled: false + # Set to the value of the Azure Active Directory Client and Tenant Ids + # respectively. These will be set as annotations on the KEDA service account. + # -- Id of Azure Active Directory Client to use for authentication with Azure Workload Identity. ([docs](https://keda.sh/docs/concepts/authentication/#azure-workload-identity)) + clientId: "" + # -- Id Azure Active Directory Tenant to use for authentication with for Azure Workload Identity. ([docs](https://keda.sh/docs/concepts/authentication/#azure-workload-identity)) + tenantId: "" + # Set to the value of the service account token expiration duration. + # This will be set as an annotation on the KEDA service account. + # -- Duration in seconds to automatically expire tokens for the service account. ([docs](https://keda.sh/docs/concepts/authentication/#azure-workload-identity)) + tokenExpiration: 3600 + aws: + irsa: + # -- Specifies whether [AWS IAM Roles for Service Accounts (IRSA)](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) is to be enabled or not. + enabled: false + # -- Sets the token audience for IRSA. + # This will be set as an annotation on the KEDA service account. + audience: "sts.amazonaws.com" + # -- Set to the value of the ARN of an IAM role with a web identity provider. + # This will be set as an annotation on the KEDA service account. + roleArn: "" + # -- Sets the use of an STS regional endpoint instead of global. + # Recommended to use regional endpoint in almost all cases. + # This will be set as an annotation on the KEDA service account. + stsRegionalEndpoints: "true" + # -- Set to the value of the service account token expiration duration. + # This will be set as an annotation on the KEDA service account. + tokenExpiration: 86400 + gcp: + # -- Set to true to enable GCP Workload Identity. + # See https://keda.sh/docs/2.10/authentication-providers/gcp-workload-identity/ + # This will be set as a annotation on the KEDA service account. + enabled: false + # -- GCP IAM Service Account Email which you would like to use for workload identity. + gcpIAMServiceAccount: "" + +# -- Set this if you are using an external scaler and want to communicate +# over TLS (recommended). This variable holds the name of the secret that +# will be mounted to the /grpccerts path on the Pod +grpcTLSCertsSecret: "" + +# -- Set this if you are using HashiCorp Vault and want to communicate +# over TLS (recommended). This variable holds the name of the secret that +# will be mounted to the /vault path on the Pod +hashiCorpVaultTLS: "" + +logging: + operator: + # -- Logging level for KEDA Operator. + # allowed values: `debug`, `info`, `error`, or an integer value greater than 0, specified as string + level: info + # -- Logging format for KEDA Operator. + # allowed values: `json` or `console` + format: console + # -- Logging time encoding for KEDA Operator. + # allowed values are `epoch`, `millis`, `nano`, `iso8601`, `rfc3339` or `rfc3339nano` + timeEncoding: rfc3339 + metricServer: + # -- Logging level for Metrics Server. + # allowed values: `0` for info, `4` for debug, or an integer value greater than 0, specified as string + level: 0 + + webhooks: + # -- Logging level for KEDA Operator. + # allowed values: `debug`, `info`, `error`, or an integer value greater than 0, specified as string + level: info + # -- Logging format for KEDA Admission webhooks. + # allowed values: `json` or `console` + format: console + # -- Logging time encoding for KEDA Operator. + # allowed values are `epoch`, `millis`, `nano`, `iso8601`, `rfc3339` or `rfc3339nano` + timeEncoding: rfc3339 + +# -- [Security context] for all containers +# @default -- [See below](#KEDA-is-secure-by-default) +securityContext: + # -- [Security context] of the operator container + # @default -- [See below](#KEDA-is-secure-by-default) + operator: + capabilities: + drop: + - ALL + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault + # -- [Security context] of the metricServer container + # @default -- [See below](#KEDA-is-secure-by-default) + metricServer: + capabilities: + drop: + - ALL + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault + # -- [Security context] of the admission webhooks container + # @default -- [See below](#KEDA-is-secure-by-default) + webhooks: + capabilities: + drop: + - ALL + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault + +# -- [Pod security context] for all pods +# @default -- [See below](#KEDA-is-secure-by-default) +podSecurityContext: + # -- [Pod security context] of the KEDA operator pod + # @default -- [See below](#KEDA-is-secure-by-default) + operator: + runAsNonRoot: true + # runAsUser: 1000 + # runAsGroup: 1000 + # fsGroup: 1000 + + # -- [Pod security context] of the KEDA metrics apiserver pod + # @default -- [See below](#KEDA-is-secure-by-default) + metricServer: + runAsNonRoot: true + # runAsUser: 1000 + # runAsGroup: 1000 + # fsGroup: 1000 + + # -- [Pod security context] of the KEDA admission webhooks + # @default -- [See below](#KEDA-is-secure-by-default) + webhooks: + runAsNonRoot: true + # runAsUser: 1000 + # runAsGroup: 1000 + # fsGroup: 1000 + +service: + # -- KEDA Metric Server service type + type: ClusterIP + # -- HTTPS port for KEDA Metric Server service + portHttps: 443 + # -- HTTPS port for KEDA Metric Server container + portHttpsTarget: 6443 + # -- Annotations to add the KEDA Metric Server service + annotations: {} + +# We provides the default values that we describe in our docs: +# https://keda.sh/docs/latest/operate/cluster/ +# If you want to specify the resources (or totally remove the defaults), change or comment the following +# lines, adjust them as necessary, or simply add the curly braces after 'operator' and/or 'metricServer' +# and remove/comment the default values +resources: + # -- Manage [resource request & limits] of KEDA operator pod + operator: + limits: + cpu: 1 + memory: 1000Mi + requests: + cpu: 100m + memory: 100Mi + # -- Manage [resource request & limits] of KEDA metrics apiserver pod + metricServer: + limits: + cpu: 1 + memory: 1000Mi + requests: + cpu: 100m + memory: 100Mi + # -- Manage [resource request & limits] of KEDA admission webhooks pod + webhooks: + limits: + cpu: 50m + memory: 100Mi + requests: + cpu: 10m + memory: 10Mi +# -- Node selector for pod scheduling ([docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/)) +nodeSelector: {} +# -- Tolerations for pod scheduling ([docs](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/)) +tolerations: [] + +topologySpreadConstraints: + # -- [Pod Topology Constraints] of KEDA operator pod + operator: [] + # -- [Pod Topology Constraints] of KEDA metrics apiserver pod + metricsServer: [] + # -- [Pod Topology Constraints] of KEDA admission webhooks pod + webhooks: [] + +# -- [Affinity] for pod scheduling for both KEDA operator and Metrics API Server +affinity: {} + # podAntiAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # - labelSelector: + # matchExpressions: + # - key: app + # operator: In + # values: + # - keda-operator + # - keda-operator-metrics-apiserver + # topologyKey: "kubernetes.io/hostname" + +# -- priorityClassName for all KEDA components +priorityClassName: "" + +## The default HTTP timeout in milliseconds that KEDA should use +## when making requests to external services. Removing this defaults to a +## reasonable default +http: + # -- The default HTTP timeout to use for all scalers that use raw HTTP clients (some scalers use SDKs to access target services. These have built-in HTTP clients, and the timeout does not necessarily apply to them) + timeout: 3000 + keepAlive: + # -- Enable HTTP connection keep alive + enabled: true + # -- The minimum TLS version to use for all scalers that use raw HTTP clients (some scalers use SDKs to access target services. These have built-in HTTP clients, and this value does not necessarily apply to them) + minTlsVersion: TLS12 + +## Extra KEDA Operator and Metrics Adapter container arguments +extraArgs: + # -- Additional KEDA Operator container arguments + keda: {} + # -- Additional Metrics Adapter container arguments + metricsAdapter: {} + +# -- Additional environment variables that will be passed onto all KEDA components +env: [] +# - name: ENV_NAME +# value: 'ENV-VALUE' + +# Extra volumes and volume mounts for the deployment. Optional. +volumes: + keda: + # -- Extra volumes for KEDA deployment + extraVolumes: [] + # -- Extra volume mounts for KEDA deployment + extraVolumeMounts: [] + + metricsApiServer: + # -- Extra volumes for metric server deployment + extraVolumes: [] + # -- Extra volume mounts for metric server deployment + extraVolumeMounts: [] + + webhooks: + # -- Extra volumes for admission webhooks deployment + extraVolumes: [] + # -- Extra volume mounts for admission webhooks deployment + extraVolumeMounts: [] + +prometheus: + metricServer: + # -- Enable metric server Prometheus metrics expose + enabled: false + # -- HTTP port used for exposing metrics server prometheus metrics + port: 8080 + # -- HTTP port name for exposing metrics server prometheus metrics + portName: metrics + serviceMonitor: + # -- Enables ServiceMonitor creation for the Prometheus Operator + enabled: false + # -- JobLabel selects the label from the associated Kubernetes service which will be used as the job label for all metrics. [ServiceMonitor Spec] + jobLabel: "" + # -- TargetLabels transfers labels from the Kubernetes `Service` onto the created metrics + targetLabels: [] + # -- PodTargetLabels transfers labels on the Kubernetes `Pod` onto the created metrics + podTargetLabels: [] + # -- Name of the service port this endpoint refers to. Mutually exclusive with targetPort + port: metrics + # -- Name or number of the target port of the Pod behind the Service, the port must be specified with container port property. Mutually exclusive with port + targetPort: "" + # -- Interval at which metrics should be scraped If not specified Prometheus’ global scrape interval is used. + interval: "" + # -- Timeout after which the scrape is ended If not specified, the Prometheus global scrape timeout is used unless it is less than Interval in which the latter is used + scrapeTimeout: "" + # -- DEPRECATED. List of expressions that define custom relabeling rules for metric server ServiceMonitor crd (prometheus operator). [RelabelConfig Spec] + relabellings: [] + # -- List of expressions that define custom relabeling rules for metric server ServiceMonitor crd (prometheus operator). [RelabelConfig Spec] + relabelings: [] + # -- Additional labels to add for metric server using ServiceMonitor crd (prometheus operator) + additionalLabels: {} + podMonitor: + # -- Enables PodMonitor creation for the Prometheus Operator + enabled: false + # -- Scraping interval for metric server using podMonitor crd (prometheus operator) + interval: "" + # -- Scraping timeout for metric server using podMonitor crd (prometheus operator) + scrapeTimeout: "" + # -- Scraping namespace for metric server using podMonitor crd (prometheus operator) + namespace: "" + # -- Additional labels to add for metric server using podMonitor crd (prometheus operator) + additionalLabels: {} + # -- List of expressions that define custom relabeling rules for metric server podMonitor crd (prometheus operator) + relabelings: [] + operator: + # -- Enable KEDA Operator prometheus metrics expose + enabled: false + # -- Port used for exposing KEDA Operator prometheus metrics + port: 8080 + serviceMonitor: + # -- Enables ServiceMonitor creation for the Prometheus Operator + enabled: false + # -- JobLabel selects the label from the associated Kubernetes service which will be used as the job label for all metrics. [ServiceMonitor Spec] + jobLabel: "" + # -- TargetLabels transfers labels from the Kubernetes `Service` onto the created metrics + targetLabels: [] + # -- PodTargetLabels transfers labels on the Kubernetes `Pod` onto the created metrics + podTargetLabels: [] + # -- Name of the service port this endpoint refers to. Mutually exclusive with targetPort + port: metrics + # -- Name or number of the target port of the Pod behind the Service, + # the port must be specified with container port property. Mutually exclusive with port + targetPort: "" + # -- Interval at which metrics should be scraped If not specified Prometheus’ global scrape interval is used. + interval: "" + # -- Timeout after which the scrape is ended If not specified, the Prometheus global scrape timeout is used unless it is less than Interval in which the latter is used + scrapeTimeout: "" + # -- DEPRECATED. List of expressions that define custom relabeling rules for metric server ServiceMonitor crd (prometheus operator). [RelabelConfig Spec] + relabellings: [] + # -- List of expressions that define custom relabeling rules for metric server ServiceMonitor crd (prometheus operator). [RelabelConfig Spec] + relabelings: [] + # -- Additional labels to add for metric server using ServiceMonitor crd (prometheus operator) + additionalLabels: {} + podMonitor: + # -- Enables PodMonitor creation for the Prometheus Operator + enabled: false + # -- Scraping interval for KEDA Operator using podMonitor crd (prometheus operator) + interval: "" + # -- Scraping timeout for KEDA Operator using podMonitor crd (prometheus operator) + scrapeTimeout: "" + # -- Scraping namespace for KEDA Operator using podMonitor crd (prometheus operator) + namespace: "" + # -- Additional labels to add for KEDA Operator using podMonitor crd (prometheus operator) + additionalLabels: {} + # -- List of expressions that define custom relabeling rules for KEDA Operator podMonitor crd (prometheus operator) + relabelings: [] + prometheusRules: + # -- Enables PrometheusRules creation for the Prometheus Operator + enabled: false + # -- Scraping namespace for KEDA Operator using prometheusRules crd (prometheus operator) + namespace: "" + # -- Additional labels to add for KEDA Operator using prometheusRules crd (prometheus operator) + additionalLabels: {} + # -- Additional alerts to add for KEDA Operator using prometheusRules crd (prometheus operator) + alerts: + [] + # - alert: KedaScalerErrors + # annotations: + # description: Keda scaledObject {{ $labels.scaledObject }} is experiencing errors with {{ $labels.scaler }} scaler + # summary: Keda Scaler {{ $labels.scaler }} Errors + # expr: sum by ( scaledObject , scaler) (rate(keda_metrics_adapter_scaler_errors[2m])) > 0 + # for: 2m + # labels: + webhooks: + # -- Enable KEDA admission webhooks prometheus metrics expose + enabled: false + # -- Port used for exposing KEDA admission webhooks prometheus metrics + port: 8080 + serviceMonitor: + # -- Enables ServiceMonitor creation for the Prometheus webhooks + enabled: false + # -- jobLabel selects the label from the associated Kubernetes service which will be used as the job label for all metrics. [ServiceMonitor Spec] + jobLabel: "" + # -- TargetLabels transfers labels from the Kubernetes `Service` onto the created metrics + targetLabels: [] + # -- PodTargetLabels transfers labels on the Kubernetes `Pod` onto the created metrics + podTargetLabels: [] + # -- Name of the service port this endpoint refers to. Mutually exclusive with targetPort + port: metrics + # -- Name or number of the target port of the Pod behind the Service, the port must be specified with container port property. Mutually exclusive with port + targetPort: "" + # -- Interval at which metrics should be scraped If not specified Prometheus’ global scrape interval is used. + interval: "" + # -- Timeout after which the scrape is ended If not specified, the Prometheus global scrape timeout is used unless it is less than Interval in which the latter is used + scrapeTimeout: "" + # -- DEPRECATED. List of expressions that define custom relabeling rules for metric server ServiceMonitor crd (prometheus operator). [RelabelConfig Spec] + relabellings: [] + # -- List of expressions that define custom relabeling rules for metric server ServiceMonitor crd (prometheus operator). [RelabelConfig Spec] + relabelings: [] + # -- Additional labels to add for metric server using ServiceMonitor crd (prometheus operator) + additionalLabels: {} + prometheusRules: + # -- Enables PrometheusRules creation for the Prometheus Operator + enabled: false + # -- Scraping namespace for KEDA admission webhooks using prometheusRules crd (prometheus operator) + namespace: "" + # -- Additional labels to add for KEDA admission webhooks using prometheusRules crd (prometheus operator) + additionalLabels: {} + # -- Additional alerts to add for KEDA admission webhooks using prometheusRules crd (prometheus operator) + alerts: [] + +certificates: + # -- Enables the self generation for KEDA TLS certificates inside KEDA operator + autoGenerated: true + # -- Secret name to be mounted with KEDA TLS certificates + secretName: kedaorg-certs + # -- Path where KEDA TLS certificates are mounted + mountPath: /certs + certManager: + # -- Enables Cert-manager for certificate management + enabled: false + # -- Generates a self-signed CA with Cert-manager. + # If generateCA is false, the secret with the CA + # has to be annotated with `cert-manager.io/allow-direct-injection: "true"` + generateCA: true + # -- Secret name where the CA is stored (generatedby cert-manager or user given) + caSecretName: "kedaorg-ca" + # -- Add labels/annotations to secrets created by Certificate resources + # [docs](https://cert-manager.io/docs/usage/certificate/#creating-certificate-resources) + secretTemplate: {} + # annotations: + # my-secret-annotation-1: "foo" + # my-secret-annotation-2: "bar" + # labels: + # my-secret-label: foo + +permissions: + metricServer: + restrict: + # -- Restrict Secret Access for Metrics Server + secret: false + operator: + restrict: + # -- Restrict Secret Access for KEDA operator + secret: false + +# -- Array of extra K8s manifests to deploy +extraObjects: [] + # - apiVersion: keda.sh/v1alpha1 + # kind: ClusterTriggerAuthentication + # metadata: + # name: aws-credentials + # namespace: keda + # spec: + # podIdentity: + # provider: aws-eks + +# -- Capability to turn on/off ASCII art in Helm installation notes +asciiArt: true \ No newline at end of file diff --git a/addons/keda/locals.tf b/addons/keda/locals.tf new file mode 100644 index 0000000..4675029 --- /dev/null +++ b/addons/keda/locals.tf @@ -0,0 +1,41 @@ +locals { + name = "keda" + + default_helm_config = { + name = try(var.keda_extra_configs.name, local.name) + chart = try(var.keda_extra_configs.chart, local.name) + repository = try(var.keda_extra_configs.repository, "https://kedacore.github.io/charts") + version = try(var.keda_extra_configs.version, "2.11.2") + namespace = try(var.keda_extra_configs.namespace, "keda") + create_namespace = try(var.keda_extra_configs.create_namespace, true) + description = "Keda helm Chart deployment configuration" + timeout = try(var.keda_extra_configs.timeout, "600") + lint = try(var.keda_extra_configs.lint, "false") + repository_key_file = try(var.keda_extra_configs.repository_key_file, "") + repository_cert_file = try(var.keda_extra_configs.repository_cert_file, "") + repository_username = try(var.keda_extra_configs.repository_username, "") + repository_password = try(var.keda_extra_configs.repository_password, "") + verify = try(var.keda_extra_configs.verify, "false") + keyring = try(var.keda_extra_configs.keyring, "") + disable_webhooks = try(var.keda_extra_configs.disable_webhooks, "false") + reuse_values = try(var.keda_extra_configs.reuse_values, "false") + reset_values = try(var.keda_extra_configs.reset_values, "false") + force_update = try(var.keda_extra_configs.force_update, "false") + recreate_pods = try(var.keda_extra_configs.recreate_pods, "false") + cleanup_on_fail = try(var.keda_extra_configs.cleanup_on_fail, "false") + max_history = try(var.keda_extra_configs.max_history, "0") + atomic = try(var.keda_extra_configs.atomic, "false") + skip_crds = try(var.keda_extra_configs.skip_crds, "false") + render_subchart_notes = try(var.keda_extra_configs.render_subchart_notes, "true") + disable_openapi_validation = try(var.keda_extra_configs.disable_openapi_validation, "false") + wait = try(var.keda_extra_configs.wait, "true") + wait_for_jobs = try(var.keda_extra_configs.wait_for_jobs, "false") + dependency_update = try(var.keda_extra_configs.dependency_update, "false") + replace = try(var.keda_extra_configs.replace, "false") + } + + helm_config = merge( + local.default_helm_config, + var.helm_config + ) +} diff --git a/addons/keda/main.tf b/addons/keda/main.tf new file mode 100644 index 0000000..d9d1876 --- /dev/null +++ b/addons/keda/main.tf @@ -0,0 +1,8 @@ +module "helm_addon" { + source = "../helm" + + manage_via_gitops = var.manage_via_gitops + helm_config = local.helm_config + addon_context = var.addon_context + +} \ No newline at end of file diff --git a/addons/keda/outputs.tf b/addons/keda/outputs.tf new file mode 100644 index 0000000..76afe98 --- /dev/null +++ b/addons/keda/outputs.tf @@ -0,0 +1,15 @@ +output "service_account" { + value = "${local.name}-sa" +} + +output "namespace" { + value = local.default_helm_config.namespace +} + +output "chart_version" { + value = local.default_helm_config.version +} + +output "repository" { + value = local.default_helm_config.repository +} \ No newline at end of file diff --git a/addons/keda/variables.tf b/addons/keda/variables.tf new file mode 100644 index 0000000..93938f8 --- /dev/null +++ b/addons/keda/variables.tf @@ -0,0 +1,34 @@ +variable "helm_config" { + description = "Helm provider config for Keda" + type = any + default = {} +} + +variable "manage_via_gitops" { + description = "Determines if the add-on should be managed via GitOps" + type = bool + default = false +} + +variable "addon_context" { + description = "Input configuration for the addon" + type = object({ + aws_caller_identity_account_id = string + aws_caller_identity_arn = string + aws_eks_cluster_endpoint = string + aws_partition_id = string + aws_region_name = string + eks_cluster_id = string + eks_oidc_issuer_url = string + eks_oidc_provider_arn = string + tags = map(string) + }) +} + + + +variable "keda_extra_configs" { + description = "Override attributes of helm_release terraform resource" + type = any + default = {} +} diff --git a/addons/keda/versions.tf b/addons/keda/versions.tf new file mode 100644 index 0000000..2d6c28d --- /dev/null +++ b/addons/keda/versions.tf @@ -0,0 +1,14 @@ +terraform { + required_version = ">= 1.0.0" + + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.10" + } + aws = { + source = "hashicorp/aws" + version = ">= 3.72" + } + } +} diff --git a/addons/kube-state-metrics/locals.tf b/addons/kube-state-metrics/locals.tf index 5dee59a..a7ff591 100644 --- a/addons/kube-state-metrics/locals.tf +++ b/addons/kube-state-metrics/locals.tf @@ -34,7 +34,6 @@ locals { replace = try(var.kube_state_metrics_extra_configs.replace, "false") } - kube_state_metrics_extra_configs = var.kube_state_metrics_extra_configs helm_config = merge( local.default_helm_config, diff --git a/main.tf b/main.tf index f3027c5..c135778 100644 --- a/main.tf +++ b/main.tf @@ -179,4 +179,13 @@ module "kube_state_metrics" { manage_via_gitops = var.manage_via_gitops addon_context = local.addon_context kube_state_metrics_extra_configs = var.kube_state_metrics_extra_configs +} + +module "keda" { + count = var.keda ? 1 : 0 + source = "./addons/keda" + helm_config = var.keda_helm_config != null ? var.keda_helm_config : { values = [local_file.keda_helm_config[count.index].content] } + manage_via_gitops = var.manage_via_gitops + addon_context = local.addon_context + keda_extra_configs = var.keda_extra_configs } \ No newline at end of file diff --git a/outputs.tf b/outputs.tf index 16e3a63..e569c0a 100644 --- a/outputs.tf +++ b/outputs.tf @@ -223,4 +223,18 @@ output "kube_state_metrics_chart_version" { output "kube_state_metrics_repository" { value = module.kube_state_metrics[*].repository description = "Helm chart repository of the Kube-State-Metrics." +} + +#----------- KEDA ------------------------ +output "keda_namespace" { + value = module.keda[*].namespace + description = "The namespace where Keda is deployed." +} +output "keda_chart_version" { + value = module.keda[*].chart_version + description = "Chart version of the Keda Helm Chart." +} +output "keda_repository" { + value = module.keda[*].repository + description = "Helm chart repository of the Keda." } \ No newline at end of file diff --git a/override_values.tf b/override_values.tf index 503d277..f5086a3 100644 --- a/override_values.tf +++ b/override_values.tf @@ -608,3 +608,29 @@ resources: EOT filename = "${path.module}/override_vales/kube_state_metrics.yaml" } + + +resource "local_file" "keda_helm_config" { + count = var.keda && (var.keda_helm_config == null) ? 1 : 0 + content = <