diff --git a/Makefile b/Makefile index 006fc800..76738863 100644 --- a/Makefile +++ b/Makefile @@ -246,7 +246,7 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified .PHONY: deploy deploy: install ## Deploy controller to the K8s cluster specified in ~/.kube/config. - $(KUSTOMIZE) build config/default | \ + $(KUSTOMIZE) build config/default/k8s | \ sed -e "s||$(OPERATOR_IMG)|g" \ -e "s||$(KEPLER_IMG)|g" \ | tee tmp/deploy.yaml | \ @@ -254,7 +254,7 @@ deploy: install ## Deploy controller to the K8s cluster specified in ~/.kube/con .PHONY: undeploy undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) build config/default | \ + $(KUSTOMIZE) build config/default/k8s | \ kubectl delete --ignore-not-found=$(ignore-not-found) -f - ##@ Build Dependencies diff --git a/README.md b/README.md index f38f83c2..ffdd389f 100644 --- a/README.md +++ b/README.md @@ -16,30 +16,33 @@ Check out the project on GitHub ➡️ [Kepler][kepler] ## Getting Started You’ll need a Kubernetes/OpenShift cluster to run against. You can use -[KIND](https://sigs.k8s.io/kind) or microshift to get a local cluster for +[KIND](https://sigs.k8s.io/kind) to get a local cluster for testing, or run against a remote cluster. **Note:** Your controller will automatically use the current context in your kubeconfig file (i.e. whatever cluster `kubectl cluster-info` shows). - ### To run a kind cluster locally ```sh make cluster-up ``` -### To run a microshift cluster locally +### Run kepler-operator locally out of cluster ```sh -make cluster-up CLUSTER_PROVIDER=microshift +make tools +make run +kubectl apply -k config/samples/ ``` -### Run kepler-operator locally out of cluster +### Run kepler-operator on vanilla Kubernetes ```sh make tools -make run +kubectl create -f https://github.com/prometheus-operator/prometheus-operator/releases/download/v0.76.0/bundle.yaml +kubectl create -f https://github.com/jetstack/cert-manager/releases/download/v1.15.3/cert-manager.yaml +make deploy kubectl apply -k config/samples/ ``` @@ -62,11 +65,13 @@ kubectl apply -k config/samples/ ### Uninstall the operator -List the installed version and the releated resources that will be +List the installed version and the related resources that will be deleted before uninstalling by running the uninstall script. + ```sh ./hack/uninstall-operator.sh ``` + Once the above is verified, uninstall the operator and all the related resources by specifying the `--delete` flag. @@ -76,6 +81,7 @@ resources by specifying the `--delete` flag. ``` ## Developer Docs + [Developer Docs][dev-docs] can be found under [docs/developer][dev-docs] ### Automated development environment @@ -94,6 +100,7 @@ documentation for more details). ## Contributing You can contribute by: + * Raising [issues](https://github.com/sustainable-computing-io/kepler-operator/issues) related to kepler-operator * Fixing issues by opening [Pull Requests](https://github.com/sustainable-computing-io/kepler-operator/pulls) diff --git a/bundle/manifests/kepler-operator.clusterserviceversion.yaml b/bundle/manifests/kepler-operator.clusterserviceversion.yaml index 6102de6d..b513c2db 100644 --- a/bundle/manifests/kepler-operator.clusterserviceversion.yaml +++ b/bundle/manifests/kepler-operator.clusterserviceversion.yaml @@ -28,7 +28,7 @@ metadata: capabilities: Seamless Upgrades categories: Monitoring containerImage: quay.io/sustainable_computing_io/kepler-operator:0.13.0 - createdAt: "2024-07-11T07:04:57Z" + createdAt: "2024-08-16T10:17:46Z" description: 'Deploys and Manages Kepler on Kubernetes ' operators.operatorframework.io/builder: operator-sdk-v1.35.0 operators.operatorframework.io/internal-objects: |- @@ -259,9 +259,9 @@ spec: containers: - args: - --openshift + - --deployment-namespace=kepler-operator - --leader-elect - --kepler.image=$(RELATED_IMAGE_KEPLER) - - --deployment-namespace=kepler-operator - --zap-log-level=5 command: - /manager diff --git a/config/default/k8s/kustomization.yaml b/config/default/k8s/kustomization.yaml new file mode 100644 index 00000000..37d4e4e3 --- /dev/null +++ b/config/default/k8s/kustomization.yaml @@ -0,0 +1,142 @@ +# Adds namespace to all resources. +namespace: kepler-operator-system + +# Value of this field is prepended to the +# names of all resources, e.g. a deployment named +# "wordpress" becomes "alices-wordpress". +# Note that it should also match with the prefix (text before '-') of the namespace +# field above. +namePrefix: kepler-operator- + +# Labels to add to all resources and selectors. +#labels: +#- includeSelectors: true +# pairs: +# someName: someValue + +resources: +- ../../crd +- ../../rbac +- ../../manager/overlays/k8s +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in +# crd/kustomization.yaml +- ../../webhook +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. +- ../../certmanager +# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. +- ../../prometheus + +patches: +# Protect the /metrics endpoint by putting it behind auth. +# If you want your controller-manager to expose the /metrics +# endpoint w/o any authn/z, please comment the following line. +# - path: manager_auth_proxy_patch.yaml + +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in +# crd/kustomization.yaml +- path: manager_webhook_patch.yaml + +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. +# Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. +# 'CERTMANAGER' needs to be enabled to use ca injection +- path: webhookcainjection_patch.yaml + +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. +# Uncomment the following replacements to add the cert-manager CA injection annotations +replacements: + - source: # Add cert-manager annotation to ValidatingWebhookConfiguration, MutatingWebhookConfiguration and CRDs + kind: Certificate + group: cert-manager.io + version: v1 + name: serving-cert # this name should match the one in certificate.yaml + fieldPath: .metadata.namespace # namespace of the certificate CR + targets: + - select: + kind: ValidatingWebhookConfiguration + fieldPaths: + - .metadata.annotations.[cert-manager.io/inject-ca-from] + options: + delimiter: '/' + index: 0 + create: true + - select: + kind: MutatingWebhookConfiguration + fieldPaths: + - .metadata.annotations.[cert-manager.io/inject-ca-from] + options: + delimiter: '/' + index: 0 + create: true + - select: + kind: CustomResourceDefinition + fieldPaths: + - .metadata.annotations.[cert-manager.io/inject-ca-from] + options: + delimiter: '/' + index: 0 + create: true + - source: + kind: Certificate + group: cert-manager.io + version: v1 + name: serving-cert # this name should match the one in certificate.yaml + fieldPath: .metadata.name + targets: + - select: + kind: ValidatingWebhookConfiguration + fieldPaths: + - .metadata.annotations.[cert-manager.io/inject-ca-from] + options: + delimiter: '/' + index: 1 + create: true + - select: + kind: MutatingWebhookConfiguration + fieldPaths: + - .metadata.annotations.[cert-manager.io/inject-ca-from] + options: + delimiter: '/' + index: 1 + create: true + - select: + kind: CustomResourceDefinition + fieldPaths: + - .metadata.annotations.[cert-manager.io/inject-ca-from] + options: + delimiter: '/' + index: 1 + create: true + - source: # Add cert-manager annotation to the webhook Service + kind: Service + version: v1 + name: webhook-service + fieldPath: .metadata.name # namespace of the service + targets: + - select: + kind: Certificate + group: cert-manager.io + version: v1 + fieldPaths: + - .spec.dnsNames.0 + - .spec.dnsNames.1 + options: + delimiter: '.' + index: 0 + create: true + - source: + kind: Service + version: v1 + name: webhook-service + fieldPath: .metadata.namespace # namespace of the service + targets: + - select: + kind: Certificate + group: cert-manager.io + version: v1 + fieldPaths: + - .spec.dnsNames.0 + - .spec.dnsNames.1 + options: + delimiter: '.' + index: 1 + create: true diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/k8s/manager_auth_proxy_patch.yaml similarity index 100% rename from config/default/manager_auth_proxy_patch.yaml rename to config/default/k8s/manager_auth_proxy_patch.yaml diff --git a/config/default/manager_config_patch.yaml b/config/default/k8s/manager_config_patch.yaml similarity index 100% rename from config/default/manager_config_patch.yaml rename to config/default/k8s/manager_config_patch.yaml diff --git a/config/default/manager_webhook_patch.yaml b/config/default/k8s/manager_webhook_patch.yaml similarity index 100% rename from config/default/manager_webhook_patch.yaml rename to config/default/k8s/manager_webhook_patch.yaml diff --git a/config/default/webhookcainjection_patch.yaml b/config/default/k8s/webhookcainjection_patch.yaml similarity index 100% rename from config/default/webhookcainjection_patch.yaml rename to config/default/k8s/webhookcainjection_patch.yaml diff --git a/config/default/kustomization.yaml b/config/default/openshift/kustomization.yaml similarity index 97% rename from config/default/kustomization.yaml rename to config/default/openshift/kustomization.yaml index 886a355e..9d67806b 100644 --- a/config/default/kustomization.yaml +++ b/config/default/openshift/kustomization.yaml @@ -15,16 +15,16 @@ namePrefix: kepler-operator- # someName: someValue resources: -- ../crd -- ../rbac -- ../manager +- ../../crd +- ../../rbac +- ../../manager/overlays/openshift # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml -- ../webhook +- ../../webhook # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. #- ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. -- ../prometheus +- ../../prometheus patches: # Protect the /metrics endpoint by putting it behind auth. diff --git a/config/default/openshift/manager_auth_proxy_patch.yaml b/config/default/openshift/manager_auth_proxy_patch.yaml new file mode 100644 index 00000000..a74efc72 --- /dev/null +++ b/config/default/openshift/manager_auth_proxy_patch.yaml @@ -0,0 +1,56 @@ +# This patch inject a sidecar container which is a HTTP proxy for the +# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - arm64 + - ppc64le + - s390x + - key: kubernetes.io/os + operator: In + values: + - linux + containers: + - name: kube-rbac-proxy + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - "ALL" + image: gcr.io/kubebuilder/kube-rbac-proxy:v0.15.0 + args: + - "--secure-listen-address=0.0.0.0:8443" + - "--upstream=http://127.0.0.1:8080/" + - "--logtostderr=true" + - "--v=0" + ports: + - containerPort: 8443 + protocol: TCP + name: https + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 5m + memory: 64Mi + - name: manager + args: + - "--health-probe-bind-address=:8081" + - "--metrics-bind-address=127.0.0.1:8080" + - "--leader-elect" + - "--zap-log-level=3" diff --git a/config/default/openshift/manager_config_patch.yaml b/config/default/openshift/manager_config_patch.yaml new file mode 100644 index 00000000..f6f58916 --- /dev/null +++ b/config/default/openshift/manager_config_patch.yaml @@ -0,0 +1,10 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager diff --git a/config/default/openshift/manager_webhook_patch.yaml b/config/default/openshift/manager_webhook_patch.yaml new file mode 100644 index 00000000..349289c3 --- /dev/null +++ b/config/default/openshift/manager_webhook_patch.yaml @@ -0,0 +1,23 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller + namespace: system +spec: + template: + spec: + containers: + - name: manager + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: webhook-server-cert diff --git a/config/default/openshift/webhookcainjection_patch.yaml b/config/default/openshift/webhookcainjection_patch.yaml new file mode 100644 index 00000000..6b105181 --- /dev/null +++ b/config/default/openshift/webhookcainjection_patch.yaml @@ -0,0 +1,29 @@ +# This patch add annotation to admission webhook config and +# CERTIFICATE_NAMESPACE and CERTIFICATE_NAME will be substituted by kustomize +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + labels: + app.kubernetes.io/name: mutatingwebhookconfiguration + app.kubernetes.io/instance: mutating-webhook-configuration + app.kubernetes.io/component: webhook + app.kubernetes.io/created-by: kepler-operator + app.kubernetes.io/part-of: kepler-operator + app.kubernetes.io/managed-by: kustomize + name: mutating-webhook-configuration + annotations: + cert-manager.io/inject-ca-from: CERTIFICATE_NAMESPACE/CERTIFICATE_NAME +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + labels: + app.kubernetes.io/name: validatingwebhookconfiguration + app.kubernetes.io/instance: validating-webhook-configuration + app.kubernetes.io/component: webhook + app.kubernetes.io/created-by: kepler-operator + app.kubernetes.io/part-of: kepler-operator + app.kubernetes.io/managed-by: kustomize + name: validating-webhook-configuration + annotations: + cert-manager.io/inject-ca-from: CERTIFICATE_NAMESPACE/CERTIFICATE_NAME diff --git a/config/manager/kustomization.yaml b/config/manager/base/kustomization.yaml similarity index 100% rename from config/manager/kustomization.yaml rename to config/manager/base/kustomization.yaml diff --git a/config/manager/manager.yaml b/config/manager/base/manager.yaml similarity index 98% rename from config/manager/manager.yaml rename to config/manager/base/manager.yaml index 19117049..1df2db5d 100644 --- a/config/manager/manager.yaml +++ b/config/manager/base/manager.yaml @@ -73,10 +73,8 @@ spec: value: '' args: # TODO: move --openshift and deployment-namespace to openshift specific kustomize directory - - --openshift - --leader-elect - --kepler.image=$(RELATED_IMAGE_KEPLER) - - --deployment-namespace=kepler-operator - --zap-log-level=5 image: '' imagePullPolicy: IfNotPresent diff --git a/config/manager/overlays/k8s/kustomization.yaml b/config/manager/overlays/k8s/kustomization.yaml new file mode 100644 index 00000000..e21f7d90 --- /dev/null +++ b/config/manager/overlays/k8s/kustomization.yaml @@ -0,0 +1,15 @@ +resources: +- ../../base + +patches: + - target: + group: apps + version: v1 + kind: Deployment + name: controller + namespace: system + patch: |- + - op: add + path: /spec/template/spec/containers/0/args/0 + value: --deployment-namespace=kepler + diff --git a/config/manager/overlays/openshift/kustomization.yaml b/config/manager/overlays/openshift/kustomization.yaml new file mode 100644 index 00000000..523dfcb7 --- /dev/null +++ b/config/manager/overlays/openshift/kustomization.yaml @@ -0,0 +1,18 @@ +resources: +- ../../base + +patches: + - target: + group: apps + version: v1 + kind: Deployment + name: controller + namespace: system + patch: |- + - op: add + path: /spec/template/spec/containers/0/args/0 + value: --openshift + - op: add + path: /spec/template/spec/containers/0/args/1 + value: --deployment-namespace=kepler-operator + diff --git a/config/manifests/kustomization.yaml b/config/manifests/kustomization.yaml index ebd941b1..5d47c2f6 100644 --- a/config/manifests/kustomization.yaml +++ b/config/manifests/kustomization.yaml @@ -2,14 +2,14 @@ # used to generate the 'manifests/' directory in a bundle. resources: - bases/kepler-operator.clusterserviceversion.yaml -- ../default +- ../default/openshift - ../samples - ../scorecard # [WEBHOOK] To enable webhooks, uncomment all the sections with [WEBHOOK] prefix. # Do NOT uncomment sections with prefix [CERTMANAGER], as OLM does not support cert-manager. # These patches remove the unnecessary "cert" volume and its manager container volumeMount. -patchesJson6902: +patches: - target: group: apps version: v1 diff --git a/hack/tools.sh b/hack/tools.sh index bfcaffd8..0c46c47b 100755 --- a/hack/tools.sh +++ b/hack/tools.sh @@ -28,7 +28,7 @@ declare -r PROJECT_ROOT GOOS GOARCH declare -r LOCAL_BIN="$PROJECT_ROOT/tmp/bin" # versions -declare -r KUSTOMIZE_VERSION=${KUSTOMIZE_VERSION:-v3.8.7} +declare -r KUSTOMIZE_VERSION=${KUSTOMIZE_VERSION:-v5.4.3} declare -r CONTROLLER_TOOLS_VERSION=${CONTROLLER_TOOLS_VERSION:-v0.13.0} declare -r OPERATOR_SDK_VERSION=${OPERATOR_SDK_VERSION:-v1.35.0} declare -r YQ_VERSION=${YQ_VERSION:-v4.34.2}